Sample stacktraces which cause the FetchetThread to crash while the broker keeps running: [ReplicaFetcher replicaId=1008, leaderId=1011, fetcherId=0] org.apache.kafka.common.KafkaException: Error processing data for partition partition-a-33 offset 28442022 at kafka.server.AbstractFetcherThread.$anonfun$processFetchRequest$7(AbstractFetcherThread.scala:208) at scala.Option.foreach(Option.scala:257) at kafka.server.AbstractFetcherThread.$anonfun$processFetchRequest$6(AbstractFetcherThread.scala:172) at kafka.server.AbstractFetcherThread.$anonfun$processFetchRequest$6$adapted(AbstractFetcherThread.scala:169) at scala.collection.mutable.ResizableArray.foreach(ResizableArray.scala:59) at scala.collection.mutable.ResizableArray.foreach$(ResizableArray.scala:52) at scala.collection.mutable.ArrayBuffer.foreach(ArrayBuffer.scala:48) at kafka.server.AbstractFetcherThread.$anonfun$processFetchRequest$5(AbstractFetcherThread.scala:169) at scala.runtime.java8.JFunction0$mcV$sp.apply(JFunction0$mcV$sp.java:12) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Caused by: kafka.common.UnexpectedAppendOffsetException: Unexpected offset in append to partition-a-33. First offset 28442021 is less than the next offset 28442022. First 10 offsets in append: List(28442021, 28442022, 28442023, 28442024, 28442025, 28442026, 28442027, 28442028, 28442029, 28442030), last offset in append: 28442043. Log start offset = 27823537 at kafka.log.Log.$anonfun$append$2(Log.scala:830) at kafka.log.Log.maybeHandleIOException(Log.scala:1837) at kafka.log.Log.append(Log.scala:752) at kafka.log.Log.appendAsFollower(Log.scala:733) at kafka.cluster.Partition.$anonfun$doAppendRecordsToFollowerOrFutureReplica$1(Partition.scala:589) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.utils.CoreUtils$.inReadLock(CoreUtils.scala:257) at kafka.cluster.Partition.doAppendRecordsToFollowerOrFutureReplica(Partition.scala:577) at kafka.cluster.Partition.appendRecordsToFollowerOrFutureReplica(Partition.scala:596) at kafka.server.ReplicaFetcherThread.processPartitionData(ReplicaFetcherThread.scala:129) at kafka.server.ReplicaFetcherThread.processPartitionData(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.$anonfun$processFetchRequest$7(AbstractFetcherThread.scala:186) ... 12 more [ReplicaFetcher replicaId=1011, leaderId=1014, fetcherId=0] Error due to org.apache.kafka.common.KafkaException: Error processing data for partition partition-b-97 offset 1388 at kafka.server.AbstractFetcherThread.$anonfun$processFetchRequest$7(AbstractFetcherThread.scala:208) at scala.Option.foreach(Option.scala:257) at kafka.server.AbstractFetcherThread.$anonfun$processFetchRequest$6(AbstractFetcherThread.scala:172) at kafka.server.AbstractFetcherThread.$anonfun$processFetchRequest$6$adapted(AbstractFetcherThread.scala:169) at scala.collection.mutable.ResizableArray.foreach(ResizableArray.scala:59) at scala.collection.mutable.ResizableArray.foreach$(ResizableArray.scala:52) at scala.collection.mutable.ArrayBuffer.foreach(ArrayBuffer.scala:48) at kafka.server.AbstractFetcherThread.$anonfun$processFetchRequest$5(AbstractFetcherThread.scala:169) at scala.runtime.java8.JFunction0$mcV$sp.apply(JFunction0$mcV$sp.java:12) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Caused by: org.apache.kafka.common.KafkaException: Trying to roll a new log segment for topic partition partition-b-97 with start offset 1388 while it already exists. at kafka.log.Log.$anonfun$roll$2(Log.scala:1499) at kafka.log.Log.maybeHandleIOException(Log.scala:1837) at kafka.log.Log.roll(Log.scala:1465) at kafka.log.Log.maybeRoll(Log.scala:1450) at kafka.log.Log.$anonfun$append$2(Log.scala:858) at kafka.log.Log.maybeHandleIOException(Log.scala:1837) at kafka.log.Log.append(Log.scala:752) at kafka.log.Log.appendAsFollower(Log.scala:733) at kafka.cluster.Partition.$anonfun$doAppendRecordsToFollowerOrFutureReplica$1(Partition.scala:589) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.utils.CoreUtils$.inReadLock(CoreUtils.scala:257) at kafka.cluster.Partition.doAppendRecordsToFollowerOrFutureReplica(Partition.scala:577) at kafka.cluster.Partition.appendRecordsToFollowerOrFutureReplica(Partition.scala:596) at kafka.server.ReplicaFetcherThread.processPartitionData(ReplicaFetcherThread.scala:129) at kafka.server.ReplicaFetcherThread.processPartitionData(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.$anonfun$processFetchRequest$7(AbstractFetcherThread.scala:186) ... 12 more [ReplicaFetcher replicaId=1006, leaderId=1014, fetcherId=0] Error due to org.apache.kafka.common.KafkaException: Error processing data for partition partition-a-121 offset 29594324 at kafka.server.AbstractFetcherThread.$anonfun$processFetchRequest$7(AbstractFetcherThread.scala:208) at scala.Option.foreach(Option.scala:257) at kafka.server.AbstractFetcherThread.$anonfun$processFetchRequest$6(AbstractFetcherThread.scala:172) at kafka.server.AbstractFetcherThread.$anonfun$processFetchRequest$6$adapted(AbstractFetcherThread.scala:169) at scala.collection.mutable.ResizableArray.foreach(ResizableArray.scala:59) at scala.collection.mutable.ResizableArray.foreach$(ResizableArray.scala:52) at scala.collection.mutable.ArrayBuffer.foreach(ArrayBuffer.scala:48) at kafka.server.AbstractFetcherThread.$anonfun$processFetchRequest$5(AbstractFetcherThread.scala:169) at scala.runtime.java8.JFunction0$mcV$sp.apply(JFunction0$mcV$sp.java:12) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Caused by: java.lang.IllegalStateException: Kafka scheduler is not running. at kafka.utils.KafkaScheduler.ensureRunning(KafkaScheduler.scala:140) at kafka.utils.KafkaScheduler.schedule(KafkaScheduler.scala:110) at kafka.log.Log.$anonfun$roll$2(Log.scala:1504) at kafka.log.Log.maybeHandleIOException(Log.scala:1837) at kafka.log.Log.roll(Log.scala:1465) at kafka.log.Log.maybeRoll(Log.scala:1450) at kafka.log.Log.$anonfun$append$2(Log.scala:858) at kafka.log.Log.maybeHandleIOException(Log.scala:1837) at kafka.log.Log.append(Log.scala:752) at kafka.log.Log.appendAsFollower(Log.scala:733) at kafka.cluster.Partition.$anonfun$doAppendRecordsToFollowerOrFutureReplica$1(Partition.scala:589) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.utils.CoreUtils$.inReadLock(CoreUtils.scala:257) at kafka.cluster.Partition.doAppendRecordsToFollowerOrFutureReplica(Partition.scala:577) at kafka.cluster.Partition.appendRecordsToFollowerOrFutureReplica(Partition.scala:596) at kafka.server.ReplicaFetcherThread.processPartitionData(ReplicaFetcherThread.scala:129) at kafka.server.ReplicaFetcherThread.processPartitionData(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.$anonfun$processFetchRequest$7(AbstractFetcherThread.scala:186) ... 12 more