2017-04-28 10:21:13 Full thread dump Java HotSpot(TM) 64-Bit Server VM (25.91-b14 mixed mode): "Attach Listener" #240 daemon prio=9 os_prio=0 tid=0x00007fb588001000 nid=0x7a76 waiting on condition [0x0000000000000000] java.lang.Thread.State: RUNNABLE "RMI Scheduler(0)" #80 daemon prio=5 os_prio=0 tid=0x00007fb408020800 nid=0x4ebc waiting on condition [0x00007fb5139d1000] java.lang.Thread.State: WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00000004002d62c8> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1081) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1067) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) at java.lang.Thread.run(Thread.java:745) "executor-Fetch" #77 prio=5 os_prio=0 tid=0x00007fb480001000 nid=0xdd6e waiting on condition [0x00007fb4cd373000] java.lang.Thread.State: WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x000000040055d6d0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1067) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) at java.lang.Thread.run(Thread.java:745) "ReplicaFetcherThread-0-74" #76 prio=5 os_prio=0 tid=0x00007fb44c01d800 nid=0xdd6a runnable [0x00007fb4cd674000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x0000000400735788> (a sun.nio.ch.Util$2) - locked <0x0000000400735778> (a java.util.Collections$UnmodifiableSet) - locked <0x0000000400735730> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:489) at org.apache.kafka.common.network.Selector.poll(Selector.java:298) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:349) at kafka.utils.NetworkClientBlockingOps$.recursivePoll$1(NetworkClientBlockingOps.scala:135) at kafka.utils.NetworkClientBlockingOps$.pollContinuously$extension(NetworkClientBlockingOps.scala:142) at kafka.utils.NetworkClientBlockingOps$.blockingSendAndReceive$extension(NetworkClientBlockingOps.scala:108) at kafka.server.ReplicaFetcherThread.sendRequest(ReplicaFetcherThread.scala:249) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:234) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:42) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:118) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:103) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:63) "kafka-scheduler-9" #75 daemon prio=5 os_prio=0 tid=0x00007fb478004000 nid=0xdae9 waiting on condition [0x00007fb4dfa1b000] java.lang.Thread.State: WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00000004000e9f20> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1088) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1067) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) at java.lang.Thread.run(Thread.java:745) "kafka-scheduler-8" #74 daemon prio=5 os_prio=0 tid=0x00007fb43404e800 nid=0xdae8 waiting on condition [0x00007fb4dfb1c000] java.lang.Thread.State: WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00000004000e9f20> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1088) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1067) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) at java.lang.Thread.run(Thread.java:745) "ReplicaFetcherThread-0-73" #73 prio=5 os_prio=0 tid=0x00007fb434043000 nid=0xdae7 runnable [0x00007fb4dfc1d000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x000000040073f010> (a sun.nio.ch.Util$2) - locked <0x000000040073f000> (a java.util.Collections$UnmodifiableSet) - locked <0x000000040073efb8> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:489) at org.apache.kafka.common.network.Selector.poll(Selector.java:298) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:349) at kafka.utils.NetworkClientBlockingOps$.recursivePoll$1(NetworkClientBlockingOps.scala:135) at kafka.utils.NetworkClientBlockingOps$.pollContinuously$extension(NetworkClientBlockingOps.scala:142) at kafka.utils.NetworkClientBlockingOps$.blockingSendAndReceive$extension(NetworkClientBlockingOps.scala:108) at kafka.server.ReplicaFetcherThread.sendRequest(ReplicaFetcherThread.scala:249) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:234) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:42) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:118) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:103) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:63) "kafka-request-handler-14" #72 daemon prio=5 os_prio=0 tid=0x00007fb6c5153000 nid=0xdae6 waiting on condition [0x00007fb4e011e000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x0000000400687dd0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.ArrayBlockingQueue.poll(ArrayBlockingQueue.java:418) at kafka.network.RequestChannel.receiveRequest(RequestChannel.scala:237) at kafka.server.KafkaRequestHandler.run(KafkaRequestHandler.scala:50) at java.lang.Thread.run(Thread.java:745) "kafka-request-handler-13" #71 daemon prio=5 os_prio=0 tid=0x00007fb6c5151800 nid=0xdae5 runnable [0x00007fb4e021e000] java.lang.Thread.State: RUNNABLE at java.util.zip.Inflater.inflateBytes(Native Method) at java.util.zip.Inflater.inflate(Inflater.java:259) - locked <0x0000000651520fd0> (a java.util.zip.ZStreamRef) at java.util.zip.InflaterInputStream.read(InflaterInputStream.java:152) at java.util.zip.GZIPInputStream.read(GZIPInputStream.java:117) at java.io.DataInputStream.readFully(DataInputStream.java:195) at org.apache.kafka.common.record.RecordsIterator$DataLogInputStream.nextEntry(RecordsIterator.java:110) at org.apache.kafka.common.record.RecordsIterator$DeepRecordsIterator.(RecordsIterator.java:162) at org.apache.kafka.common.record.RecordsIterator.makeNext(RecordsIterator.java:79) at org.apache.kafka.common.record.RecordsIterator.makeNext(RecordsIterator.java:34) at org.apache.kafka.common.utils.AbstractIterator.maybeComputeNext(AbstractIterator.java:79) at org.apache.kafka.common.utils.AbstractIterator.hasNext(AbstractIterator.java:45) at scala.collection.convert.Wrappers$JIteratorWrapper.hasNext(Wrappers.scala:39) at scala.collection.Iterator.foreach(Iterator.scala:929) at scala.collection.Iterator.foreach$(Iterator.scala:929) at scala.collection.AbstractIterator.foreach(Iterator.scala:1406) at scala.collection.IterableLike.foreach(IterableLike.scala:71) at scala.collection.IterableLike.foreach$(IterableLike.scala:70) at scala.collection.AbstractIterable.foreach(Iterable.scala:54) at kafka.log.LogValidator$.validateMessagesAndAssignOffsetsCompressed(LogValidator.scala:167) at kafka.log.LogValidator$.validateMessagesAndAssignOffsets(LogValidator.scala:68) at kafka.log.Log.liftedTree1$1(Log.scala:380) at kafka.log.Log.append(Log.scala:371) - locked <0x0000000400423858> (a java.lang.Object) at kafka.cluster.Partition.$anonfun$appendRecordsToLeader$1(Partition.scala:451) at kafka.cluster.Partition$$Lambda$880/1108169257.apply(Unknown Source) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:213) at kafka.utils.CoreUtils$.inReadLock(CoreUtils.scala:219) at kafka.cluster.Partition.appendRecordsToLeader(Partition.scala:439) at kafka.server.ReplicaManager.$anonfun$appendToLocalLog$2(ReplicaManager.scala:389) at kafka.server.ReplicaManager$$Lambda$879/989434829.apply(Unknown Source) at scala.collection.TraversableLike.$anonfun$map$1(TraversableLike.scala:234) at scala.collection.TraversableLike$$Lambda$10/315138752.apply(Unknown Source) at scala.collection.Iterator.foreach(Iterator.scala:929) at scala.collection.Iterator.foreach$(Iterator.scala:929) at scala.collection.AbstractIterator.foreach(Iterator.scala:1406) at scala.collection.IterableLike.foreach(IterableLike.scala:71) at scala.collection.IterableLike.foreach$(IterableLike.scala:70) at scala.collection.AbstractIterable.foreach(Iterable.scala:54) at scala.collection.TraversableLike.map(TraversableLike.scala:234) at scala.collection.TraversableLike.map$(TraversableLike.scala:227) at scala.collection.AbstractTraversable.map(Traversable.scala:104) at kafka.server.ReplicaManager.appendToLocalLog(ReplicaManager.scala:375) at kafka.server.ReplicaManager.appendRecords(ReplicaManager.scala:312) at kafka.server.KafkaApis.handleProducerRequest(KafkaApis.scala:432) at kafka.server.KafkaApis.handle(KafkaApis.scala:80) at kafka.server.KafkaRequestHandler.run(KafkaRequestHandler.scala:62) at java.lang.Thread.run(Thread.java:745) "kafka-request-handler-12" #70 daemon prio=5 os_prio=0 tid=0x00007fb6c514f800 nid=0xdae4 waiting on condition [0x00007fb4ea5f6000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x0000000400687dd0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.ArrayBlockingQueue.poll(ArrayBlockingQueue.java:418) at kafka.network.RequestChannel.receiveRequest(RequestChannel.scala:237) at kafka.server.KafkaRequestHandler.run(KafkaRequestHandler.scala:50) at java.lang.Thread.run(Thread.java:745) "kafka-request-handler-11" #69 daemon prio=5 os_prio=0 tid=0x00007fb6c514e000 nid=0xdae3 waiting on condition [0x00007fb4ea6f7000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x0000000400687dd0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.ArrayBlockingQueue.poll(ArrayBlockingQueue.java:418) at kafka.network.RequestChannel.receiveRequest(RequestChannel.scala:237) at kafka.server.KafkaRequestHandler.run(KafkaRequestHandler.scala:50) at java.lang.Thread.run(Thread.java:745) "kafka-request-handler-10" #68 daemon prio=5 os_prio=0 tid=0x00007fb6c514c000 nid=0xdae2 waiting on condition [0x00007fb4ea7f8000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x0000000400687dd0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.ArrayBlockingQueue.poll(ArrayBlockingQueue.java:418) at kafka.network.RequestChannel.receiveRequest(RequestChannel.scala:237) at kafka.server.KafkaRequestHandler.run(KafkaRequestHandler.scala:50) at java.lang.Thread.run(Thread.java:745) "kafka-request-handler-9" #67 daemon prio=5 os_prio=0 tid=0x00007fb6c514a800 nid=0xdae1 waiting on condition [0x00007fb4ea8f9000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x0000000400687dd0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.ArrayBlockingQueue.poll(ArrayBlockingQueue.java:418) at kafka.network.RequestChannel.receiveRequest(RequestChannel.scala:237) at kafka.server.KafkaRequestHandler.run(KafkaRequestHandler.scala:50) at java.lang.Thread.run(Thread.java:745) "kafka-request-handler-8" #66 daemon prio=5 os_prio=0 tid=0x00007fb6c5148800 nid=0xdae0 waiting on condition [0x00007fb4ea9fa000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x0000000400687dd0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.ArrayBlockingQueue.poll(ArrayBlockingQueue.java:418) at kafka.network.RequestChannel.receiveRequest(RequestChannel.scala:237) at kafka.server.KafkaRequestHandler.run(KafkaRequestHandler.scala:50) at java.lang.Thread.run(Thread.java:745) "kafka-request-handler-7" #65 daemon prio=5 os_prio=0 tid=0x00007fb6c5147000 nid=0xdadf waiting on condition [0x00007fb4eaafb000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x0000000400687dd0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.ArrayBlockingQueue.poll(ArrayBlockingQueue.java:418) at kafka.network.RequestChannel.receiveRequest(RequestChannel.scala:237) at kafka.server.KafkaRequestHandler.run(KafkaRequestHandler.scala:50) at java.lang.Thread.run(Thread.java:745) "kafka-request-handler-6" #64 daemon prio=5 os_prio=0 tid=0x00007fb6c5145000 nid=0xdade waiting on condition [0x00007fb4eabfc000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x0000000400687dd0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.ArrayBlockingQueue.poll(ArrayBlockingQueue.java:418) at kafka.network.RequestChannel.receiveRequest(RequestChannel.scala:237) at kafka.server.KafkaRequestHandler.run(KafkaRequestHandler.scala:50) at java.lang.Thread.run(Thread.java:745) "kafka-request-handler-5" #63 daemon prio=5 os_prio=0 tid=0x00007fb6c5143800 nid=0xdadd waiting on condition [0x00007fb4eacfd000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x0000000400687dd0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.ArrayBlockingQueue.poll(ArrayBlockingQueue.java:418) at kafka.network.RequestChannel.receiveRequest(RequestChannel.scala:237) at kafka.server.KafkaRequestHandler.run(KafkaRequestHandler.scala:50) at java.lang.Thread.run(Thread.java:745) "kafka-request-handler-4" #62 daemon prio=5 os_prio=0 tid=0x00007fb6c5141800 nid=0xdadc waiting on condition [0x00007fb4eadfe000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x0000000400687dd0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.ArrayBlockingQueue.poll(ArrayBlockingQueue.java:418) at kafka.network.RequestChannel.receiveRequest(RequestChannel.scala:237) at kafka.server.KafkaRequestHandler.run(KafkaRequestHandler.scala:50) at java.lang.Thread.run(Thread.java:745) "kafka-request-handler-3" #61 daemon prio=5 os_prio=0 tid=0x00007fb6c5140000 nid=0xdadb waiting on condition [0x00007fb4f44cc000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x0000000400687dd0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.ArrayBlockingQueue.poll(ArrayBlockingQueue.java:418) at kafka.network.RequestChannel.receiveRequest(RequestChannel.scala:237) at kafka.server.KafkaRequestHandler.run(KafkaRequestHandler.scala:50) at java.lang.Thread.run(Thread.java:745) "kafka-request-handler-2" #60 daemon prio=5 os_prio=0 tid=0x00007fb6c513e000 nid=0xdada waiting on condition [0x00007fb4f45cd000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x0000000400687dd0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.ArrayBlockingQueue.poll(ArrayBlockingQueue.java:418) at kafka.network.RequestChannel.receiveRequest(RequestChannel.scala:237) at kafka.server.KafkaRequestHandler.run(KafkaRequestHandler.scala:50) at java.lang.Thread.run(Thread.java:745) "kafka-request-handler-1" #59 daemon prio=5 os_prio=0 tid=0x00007fb6c4940800 nid=0xdad9 runnable [0x00007fb4f46ce000] java.lang.Thread.State: RUNNABLE at java.util.zip.Deflater.deflateBytes(Native Method) at java.util.zip.Deflater.deflate(Deflater.java:444) - locked <0x00000006521de350> (a java.util.zip.ZStreamRef) at java.util.zip.Deflater.deflate(Deflater.java:366) at java.util.zip.DeflaterOutputStream.deflate(DeflaterOutputStream.java:251) at java.util.zip.DeflaterOutputStream.write(DeflaterOutputStream.java:211) at java.util.zip.GZIPOutputStream.write(GZIPOutputStream.java:145) - locked <0x00000006521de2c0> (a java.util.zip.GZIPOutputStream) at java.util.zip.DeflaterOutputStream.write(DeflaterOutputStream.java:188) at java.io.DataOutputStream.writeInt(DataOutputStream.java:198) at org.apache.kafka.common.record.LogEntry.writeHeader(LogEntry.java:143) at org.apache.kafka.common.record.MemoryRecordsBuilder.appendUnchecked(MemoryRecordsBuilder.java:302) at org.apache.kafka.common.record.MemoryRecordsBuilder.appendWithOffset(MemoryRecordsBuilder.java:324) at org.apache.kafka.common.record.MemoryRecords.builderWithEntries(MemoryRecords.java:411) at org.apache.kafka.common.record.MemoryRecords.builderWithEntries(MemoryRecords.java:393) at org.apache.kafka.common.record.MemoryRecords.withLogEntries(MemoryRecords.java:378) at org.apache.kafka.common.record.MemoryRecords.withLogEntries(MemoryRecords.java:337) at org.apache.kafka.common.record.AbstractRecords.toMessageFormat(AbstractRecords.java:79) at kafka.server.KafkaApis.$anonfun$handleFetchRequest$5(KafkaApis.scala:481) at kafka.server.KafkaApis$$Lambda$914/1797705547.apply(Unknown Source) at scala.collection.TraversableLike.$anonfun$map$1(TraversableLike.scala:234) at scala.collection.TraversableLike$$Lambda$10/315138752.apply(Unknown Source) at scala.collection.mutable.ResizableArray.foreach(ResizableArray.scala:59) at scala.collection.mutable.ResizableArray.foreach$(ResizableArray.scala:52) at scala.collection.mutable.ArrayBuffer.foreach(ArrayBuffer.scala:48) at scala.collection.TraversableLike.map(TraversableLike.scala:234) at scala.collection.TraversableLike.map$(TraversableLike.scala:227) at scala.collection.AbstractTraversable.map(Traversable.scala:104) at kafka.server.KafkaApis.sendResponseCallback$3(KafkaApis.scala:468) at kafka.server.KafkaApis.$anonfun$handleFetchRequest$13(KafkaApis.scala:538) at kafka.server.KafkaApis.$anonfun$handleFetchRequest$13$adapted(KafkaApis.scala:538) at kafka.server.KafkaApis$$Lambda$824/113902044.apply(Unknown Source) at kafka.server.ReplicaManager.fetchMessages(ReplicaManager.scala:478) at kafka.server.KafkaApis.handleFetchRequest(KafkaApis.scala:538) at kafka.server.KafkaApis.handle(KafkaApis.scala:81) at kafka.server.KafkaRequestHandler.run(KafkaRequestHandler.scala:62) at java.lang.Thread.run(Thread.java:745) "kafka-request-handler-0" #58 daemon prio=5 os_prio=0 tid=0x00007fb6c493f000 nid=0xdad8 waiting on condition [0x00007fb4f47cf000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x0000000400687dd0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.ArrayBlockingQueue.poll(ArrayBlockingQueue.java:418) at kafka.network.RequestChannel.receiveRequest(RequestChannel.scala:237) at kafka.server.KafkaRequestHandler.run(KafkaRequestHandler.scala:50) at java.lang.Thread.run(Thread.java:745) "group-metadata-manager-0" #57 daemon prio=5 os_prio=0 tid=0x00007fb6c490a800 nid=0xdad7 waiting on condition [0x00007fb4f48d0000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00000004005629a8> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1093) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1067) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) at java.lang.Thread.run(Thread.java:745) "ExpirationReaper-72" #56 prio=5 os_prio=0 tid=0x00007fb6c48fc000 nid=0xdad6 waiting on condition [0x00007fb4f49d1000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x000000040055e508> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.DelayQueue.poll(DelayQueue.java:259) at kafka.utils.timer.SystemTimer.advanceClock(Timer.scala:105) at kafka.server.DelayedOperationPurgatory.advanceClock(DelayedOperation.scala:354) at kafka.server.DelayedOperationPurgatory$ExpiredOperationReaper.doWork(DelayedOperation.scala:378) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:63) "ExpirationReaper-72" #55 prio=5 os_prio=0 tid=0x00007fb6c48fb800 nid=0xdad5 waiting on condition [0x00007fb4f4ad2000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x0000000400562c10> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.DelayQueue.poll(DelayQueue.java:259) at kafka.utils.timer.SystemTimer.advanceClock(Timer.scala:105) at kafka.server.DelayedOperationPurgatory.advanceClock(DelayedOperation.scala:354) at kafka.server.DelayedOperationPurgatory$ExpiredOperationReaper.doWork(DelayedOperation.scala:378) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:63) "ExpirationReaper-72" #54 prio=5 os_prio=0 tid=0x00007fb6c48f6800 nid=0xdad4 waiting on condition [0x00007fb4fc7c4000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x0000000400565200> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.DelayQueue.poll(DelayQueue.java:259) at kafka.utils.timer.SystemTimer.advanceClock(Timer.scala:105) at kafka.server.DelayedOperationPurgatory.advanceClock(DelayedOperation.scala:354) at kafka.server.DelayedOperationPurgatory$ExpiredOperationReaper.doWork(DelayedOperation.scala:378) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:63) "kafka-scheduler-7" #53 daemon prio=5 os_prio=0 tid=0x00007fb4ac002000 nid=0xdad3 waiting on condition [0x00007fb4b5617000] java.lang.Thread.State: WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00000004000e9f20> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1088) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1067) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) at java.lang.Thread.run(Thread.java:745) "kafka-scheduler-6" #52 daemon prio=5 os_prio=0 tid=0x00007fb4a8001800 nid=0xdad2 waiting on condition [0x00007fb4b6c0f000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00000004000e9f20> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1093) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1067) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) at java.lang.Thread.run(Thread.java:745) "kafka-scheduler-5" #51 daemon prio=5 os_prio=0 tid=0x00007fb6c5f41000 nid=0xdad1 waiting on condition [0x00007fb4b6d10000] java.lang.Thread.State: WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00000004000e9f20> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1088) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1067) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) at java.lang.Thread.run(Thread.java:745) "kafka-scheduler-4" #50 daemon prio=5 os_prio=0 tid=0x00007fb6c5f3f000 nid=0xdad0 waiting on condition [0x00007fb4b6e11000] java.lang.Thread.State: WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00000004000e9f20> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1088) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1067) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) at java.lang.Thread.run(Thread.java:745) "metrics-meter-tick-thread-2" #49 daemon prio=5 os_prio=0 tid=0x00007fb6c5f3d000 nid=0xdacf waiting on condition [0x00007fb4b6f12000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x000000040054a590> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1093) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1067) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) at java.lang.Thread.run(Thread.java:745) "ExpirationReaper-72" #48 prio=5 os_prio=0 tid=0x00007fb6c5f38800 nid=0xdace waiting on condition [0x00007fb4b7013000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x0000000400560968> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.DelayQueue.poll(DelayQueue.java:273) at kafka.utils.timer.SystemTimer.advanceClock(Timer.scala:105) at kafka.server.DelayedOperationPurgatory.advanceClock(DelayedOperation.scala:354) at kafka.server.DelayedOperationPurgatory$ExpiredOperationReaper.doWork(DelayedOperation.scala:378) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:63) "ExpirationReaper-72" #47 prio=5 os_prio=0 tid=0x00007fb6c5f37800 nid=0xdacd waiting on condition [0x00007fb4b7114000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x0000000400563018> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.DelayQueue.poll(DelayQueue.java:268) at kafka.utils.timer.SystemTimer.advanceClock(Timer.scala:105) at kafka.server.DelayedOperationPurgatory.advanceClock(DelayedOperation.scala:354) at kafka.server.DelayedOperationPurgatory$ExpiredOperationReaper.doWork(DelayedOperation.scala:378) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:63) "kafka-socket-acceptor-ListenerName(PLAINTEXT)-PLAINTEXT-9092" #46 prio=5 os_prio=0 tid=0x00007fb6c546a800 nid=0xdacc runnable [0x00007fb4b7215000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x0000000400560c38> (a sun.nio.ch.Util$2) - locked <0x0000000400560c28> (a java.util.Collections$UnmodifiableSet) - locked <0x0000000400560be0> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at kafka.network.Acceptor.run(SocketServer.scala:274) at java.lang.Thread.run(Thread.java:745) "kafka-network-thread-72-ListenerName(PLAINTEXT)-PLAINTEXT-4" #45 prio=5 os_prio=0 tid=0x00007fb6c5468800 nid=0xdacb runnable [0x00007fb4b7316000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x000000040055c5a0> (a sun.nio.ch.Util$2) - locked <0x000000040055c590> (a java.util.Collections$UnmodifiableSet) - locked <0x000000040055c548> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:489) at org.apache.kafka.common.network.Selector.poll(Selector.java:298) at kafka.network.Processor.poll(SocketServer.scala:494) at kafka.network.Processor.run(SocketServer.scala:432) at java.lang.Thread.run(Thread.java:745) "kafka-network-thread-72-ListenerName(PLAINTEXT)-PLAINTEXT-3" #44 prio=5 os_prio=0 tid=0x00007fb6c5467000 nid=0xdaca runnable [0x00007fb4b7417000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00000004006a7c28> (a sun.nio.ch.Util$2) - locked <0x00000004006a7c18> (a java.util.Collections$UnmodifiableSet) - locked <0x000000040066f8c8> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:489) at org.apache.kafka.common.network.Selector.poll(Selector.java:298) at kafka.network.Processor.poll(SocketServer.scala:494) at kafka.network.Processor.run(SocketServer.scala:432) at java.lang.Thread.run(Thread.java:745) "kafka-network-thread-72-ListenerName(PLAINTEXT)-PLAINTEXT-2" #43 prio=5 os_prio=0 tid=0x00007fb6c549d000 nid=0xdac9 runnable [0x00007fb4f3bcb000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x000000040059c460> (a sun.nio.ch.Util$2) - locked <0x0000000400560e40> (a java.util.Collections$UnmodifiableSet) - locked <0x000000040059c2c8> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:489) at org.apache.kafka.common.network.Selector.poll(Selector.java:298) at kafka.network.Processor.poll(SocketServer.scala:494) at kafka.network.Processor.run(SocketServer.scala:432) at java.lang.Thread.run(Thread.java:745) "kafka-network-thread-72-ListenerName(PLAINTEXT)-PLAINTEXT-1" #42 prio=5 os_prio=0 tid=0x00007fb6c549b800 nid=0xdac8 runnable [0x00007fb4f3ccc000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00000004005654d0> (a sun.nio.ch.Util$2) - locked <0x00000004005654c0> (a java.util.Collections$UnmodifiableSet) - locked <0x0000000400565478> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:489) at org.apache.kafka.common.network.Selector.poll(Selector.java:298) at kafka.network.Processor.poll(SocketServer.scala:494) at kafka.network.Processor.run(SocketServer.scala:432) at java.lang.Thread.run(Thread.java:745) "kafka-network-thread-72-ListenerName(PLAINTEXT)-PLAINTEXT-0" #41 prio=5 os_prio=0 tid=0x00007fb6c549a000 nid=0xdac7 runnable [0x00007fb4f3dcd000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x000000040055aa18> (a sun.nio.ch.Util$2) - locked <0x000000040055aa08> (a java.util.Collections$UnmodifiableSet) - locked <0x000000040055a9c0> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:489) at org.apache.kafka.common.network.Selector.poll(Selector.java:298) at kafka.network.Processor.poll(SocketServer.scala:494) at kafka.network.Processor.run(SocketServer.scala:432) at java.lang.Thread.run(Thread.java:745) "kafka-log-cleaner-thread-0" #36 prio=5 os_prio=0 tid=0x00007fb6c5416800 nid=0xdac6 waiting on condition [0x00007fb4f3ece000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x0000000400558318> (a java.util.concurrent.CountDownLatch$Sync) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer.doAcquireSharedNanos(AbstractQueuedSynchronizer.java:1037) at java.util.concurrent.locks.AbstractQueuedSynchronizer.tryAcquireSharedNanos(AbstractQueuedSynchronizer.java:1328) at java.util.concurrent.CountDownLatch.await(CountDownLatch.java:277) at kafka.log.LogCleaner$CleanerThread.cleanOrSleep(LogCleaner.scala:261) at kafka.log.LogCleaner$CleanerThread.doWork(LogCleaner.scala:220) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:63) "kafka-scheduler-3" #40 daemon prio=5 os_prio=0 tid=0x00007fb6c5411800 nid=0xdac5 waiting on condition [0x00007fb4f3fcf000] java.lang.Thread.State: WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00000004000e9f20> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1088) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1067) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) at java.lang.Thread.run(Thread.java:745) "kafka-scheduler-2" #39 daemon prio=5 os_prio=0 tid=0x00007fb6c540f800 nid=0xdac4 waiting on condition [0x00007fb4f40d0000] java.lang.Thread.State: WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00000004000e9f20> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1088) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1067) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) at java.lang.Thread.run(Thread.java:745) "kafka-scheduler-1" #38 daemon prio=5 os_prio=0 tid=0x00007fb6c540e000 nid=0xdac3 waiting on condition [0x00007fb4f41d1000] java.lang.Thread.State: WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00000004000e9f20> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1088) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1067) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) at java.lang.Thread.run(Thread.java:745) "kafka-scheduler-0" #37 daemon prio=5 os_prio=0 tid=0x00007fb6c540a800 nid=0xdac2 waiting on condition [0x00007fb4f42d2000] java.lang.Thread.State: WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00000004000e9f20> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1088) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1067) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) at java.lang.Thread.run(Thread.java:745) "metrics-meter-tick-thread-1" #35 daemon prio=5 os_prio=0 tid=0x00007fb6c4388000 nid=0xdac1 waiting on condition [0x00007fb59493b000] java.lang.Thread.State: WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x000000040054a590> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1088) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1067) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) at java.lang.Thread.run(Thread.java:745) "ThrottledRequestReaper-Produce" #33 prio=5 os_prio=0 tid=0x00007fb6c6224000 nid=0xc008 waiting on condition [0x00007fb594c3c000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x0000000400338348> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.DelayQueue.poll(DelayQueue.java:259) at kafka.server.ClientQuotaManager$ThrottledRequestReaper.doWork(ClientQuotaManager.scala:158) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:63) "ThrottledRequestReaper-Fetch" #32 prio=5 os_prio=0 tid=0x00007fb6c6222000 nid=0xc007 waiting on condition [0x00007fb594d3d000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x0000000400338530> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.DelayQueue.poll(DelayQueue.java:259) at kafka.server.ClientQuotaManager$ThrottledRequestReaper.doWork(ClientQuotaManager.scala:158) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:63) "SensorExpiryThread" #31 daemon prio=5 os_prio=0 tid=0x00007fb6c621a000 nid=0xc006 waiting on condition [0x00007fb594e3e000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00000004003386f0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1093) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1067) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) at java.lang.Thread.run(Thread.java:745) "main-EventThread" #30 daemon prio=5 os_prio=0 tid=0x00007fb6c6187000 nid=0xc005 waiting on condition [0x00007fb59513f000] java.lang.Thread.State: WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x0000000400001a50> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) at org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:501) "main-SendThread(10.52.208.73:2181)" #29 daemon prio=5 os_prio=0 tid=0x00007fb6c618f800 nid=0xc004 runnable [0x00007fb595240000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x0000000400001d08> (a sun.nio.ch.Util$2) - locked <0x0000000400001d18> (a java.util.Collections$UnmodifiableSet) - locked <0x0000000400001cc0> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.zookeeper.ClientCnxnSocketNIO.doTransport(ClientCnxnSocketNIO.java:349) at org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1141) "ZkClient-EventThread-28-10.52.208.72:2181,10.52.208.73:2181,10.52.208.74:2181" #28 daemon prio=5 os_prio=0 tid=0x00007fb6c617b800 nid=0xc003 waiting on condition [0x00007fb595341000] java.lang.Thread.State: WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00000004000012c0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) at org.I0Itec.zkclient.ZkEventThread.run(ZkEventThread.java:68) "RMI TCP Accept-0" #25 daemon prio=5 os_prio=0 tid=0x00007fb6c46e4800 nid=0xc001 runnable [0x00007fb595b43000] java.lang.Thread.State: RUNNABLE at java.net.PlainSocketImpl.socketAccept(Native Method) at java.net.AbstractPlainSocketImpl.accept(AbstractPlainSocketImpl.java:409) at java.net.ServerSocket.implAccept(ServerSocket.java:545) at java.net.ServerSocket.accept(ServerSocket.java:513) at sun.management.jmxremote.LocalRMIServerSocketFactory$1.accept(LocalRMIServerSocketFactory.java:52) at sun.rmi.transport.tcp.TCPTransport$AcceptLoop.executeAcceptLoop(TCPTransport.java:400) at sun.rmi.transport.tcp.TCPTransport$AcceptLoop.run(TCPTransport.java:372) at java.lang.Thread.run(Thread.java:745) "RMI TCP Accept-9999" #24 daemon prio=5 os_prio=0 tid=0x00007fb6c46ce800 nid=0xc000 runnable [0x00007fb595c44000] java.lang.Thread.State: RUNNABLE at java.net.PlainSocketImpl.socketAccept(Native Method) at java.net.AbstractPlainSocketImpl.accept(AbstractPlainSocketImpl.java:409) at java.net.ServerSocket.implAccept(ServerSocket.java:545) at java.net.ServerSocket.accept(ServerSocket.java:513) at sun.rmi.transport.tcp.TCPTransport$AcceptLoop.executeAcceptLoop(TCPTransport.java:400) at sun.rmi.transport.tcp.TCPTransport$AcceptLoop.run(TCPTransport.java:372) at java.lang.Thread.run(Thread.java:745) "RMI TCP Accept-0" #23 daemon prio=5 os_prio=0 tid=0x00007fb6c46b9800 nid=0xbfff runnable [0x00007fb595d45000] java.lang.Thread.State: RUNNABLE at java.net.PlainSocketImpl.socketAccept(Native Method) at java.net.AbstractPlainSocketImpl.accept(AbstractPlainSocketImpl.java:409) at java.net.ServerSocket.implAccept(ServerSocket.java:545) at java.net.ServerSocket.accept(ServerSocket.java:513) at sun.rmi.transport.tcp.TCPTransport$AcceptLoop.executeAcceptLoop(TCPTransport.java:400) at sun.rmi.transport.tcp.TCPTransport$AcceptLoop.run(TCPTransport.java:372) at java.lang.Thread.run(Thread.java:745) "Service Thread" #21 daemon prio=9 os_prio=0 tid=0x00007fb6c43a3000 nid=0xbffc runnable [0x0000000000000000] java.lang.Thread.State: RUNNABLE "C1 CompilerThread14" #20 daemon prio=9 os_prio=0 tid=0x00007fb6c439f800 nid=0xbffb waiting on condition [0x0000000000000000] java.lang.Thread.State: RUNNABLE "C1 CompilerThread13" #19 daemon prio=9 os_prio=0 tid=0x00007fb6c439d800 nid=0xbffa waiting on condition [0x0000000000000000] java.lang.Thread.State: RUNNABLE "C1 CompilerThread12" #18 daemon prio=9 os_prio=0 tid=0x00007fb6c439b800 nid=0xbff9 waiting on condition [0x0000000000000000] java.lang.Thread.State: RUNNABLE "C1 CompilerThread11" #17 daemon prio=9 os_prio=0 tid=0x00007fb6c4399000 nid=0xbff8 waiting on condition [0x0000000000000000] java.lang.Thread.State: RUNNABLE "C1 CompilerThread10" #16 daemon prio=9 os_prio=0 tid=0x00007fb6c4397800 nid=0xbff7 waiting on condition [0x0000000000000000] java.lang.Thread.State: RUNNABLE "C2 CompilerThread9" #15 daemon prio=9 os_prio=0 tid=0x00007fb6c4394800 nid=0xbff6 waiting on condition [0x0000000000000000] java.lang.Thread.State: RUNNABLE "C2 CompilerThread8" #14 daemon prio=9 os_prio=0 tid=0x00007fb6c4392800 nid=0xbff5 waiting on condition [0x0000000000000000] java.lang.Thread.State: RUNNABLE "C2 CompilerThread7" #13 daemon prio=9 os_prio=0 tid=0x00007fb6c4390800 nid=0xbff4 waiting on condition [0x0000000000000000] java.lang.Thread.State: RUNNABLE "C2 CompilerThread6" #12 daemon prio=9 os_prio=0 tid=0x00007fb6c438e800 nid=0xbff3 waiting on condition [0x0000000000000000] java.lang.Thread.State: RUNNABLE "C2 CompilerThread5" #11 daemon prio=9 os_prio=0 tid=0x00007fb6c438c000 nid=0xbff2 waiting on condition [0x0000000000000000] java.lang.Thread.State: RUNNABLE "C2 CompilerThread4" #10 daemon prio=9 os_prio=0 tid=0x00007fb6c438a000 nid=0xbff1 waiting on condition [0x0000000000000000] java.lang.Thread.State: RUNNABLE "C2 CompilerThread3" #9 daemon prio=9 os_prio=0 tid=0x00007fb6c437f800 nid=0xbff0 waiting on condition [0x0000000000000000] java.lang.Thread.State: RUNNABLE "C2 CompilerThread2" #8 daemon prio=9 os_prio=0 tid=0x00007fb6c437d800 nid=0xbfef waiting on condition [0x0000000000000000] java.lang.Thread.State: RUNNABLE "C2 CompilerThread1" #7 daemon prio=9 os_prio=0 tid=0x00007fb6c437b800 nid=0xbfee waiting on condition [0x0000000000000000] java.lang.Thread.State: RUNNABLE "C2 CompilerThread0" #6 daemon prio=9 os_prio=0 tid=0x00007fb6c4379000 nid=0xbfed waiting on condition [0x0000000000000000] java.lang.Thread.State: RUNNABLE "Signal Dispatcher" #5 daemon prio=9 os_prio=0 tid=0x00007fb6c4377000 nid=0xbfec runnable [0x0000000000000000] java.lang.Thread.State: RUNNABLE "Surrogate Locker Thread (Concurrent GC)" #4 daemon prio=9 os_prio=0 tid=0x00007fb6c4375800 nid=0xbfeb waiting on condition [0x0000000000000000] java.lang.Thread.State: RUNNABLE "Finalizer" #3 daemon prio=8 os_prio=0 tid=0x00007fb6c4342800 nid=0xbfea in Object.wait() [0x00007fb5a960a000] java.lang.Thread.State: WAITING (on object monitor) at java.lang.Object.wait(Native Method) at java.lang.ref.ReferenceQueue.remove(ReferenceQueue.java:143) - locked <0x0000000400007f98> (a java.lang.ref.ReferenceQueue$Lock) at java.lang.ref.ReferenceQueue.remove(ReferenceQueue.java:164) at java.lang.ref.Finalizer$FinalizerThread.run(Finalizer.java:209) "Reference Handler" #2 daemon prio=10 os_prio=0 tid=0x00007fb6c433e000 nid=0xbfe9 in Object.wait() [0x00007fb5a970b000] java.lang.Thread.State: WAITING (on object monitor) at java.lang.Object.wait(Native Method) at java.lang.Object.wait(Object.java:502) at java.lang.ref.Reference.tryHandlePending(Reference.java:191) - locked <0x0000000400007028> (a java.lang.ref.Reference$Lock) at java.lang.ref.Reference$ReferenceHandler.run(Reference.java:153) "main" #1 prio=5 os_prio=0 tid=0x00007fb6c400d000 nid=0xbfb1 waiting on condition [0x00007fb6c9939000] java.lang.Thread.State: WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x0000000400563188> (a java.util.concurrent.CountDownLatch$Sync) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer.parkAndCheckInterrupt(AbstractQueuedSynchronizer.java:836) at java.util.concurrent.locks.AbstractQueuedSynchronizer.doAcquireSharedInterruptibly(AbstractQueuedSynchronizer.java:997) at java.util.concurrent.locks.AbstractQueuedSynchronizer.acquireSharedInterruptibly(AbstractQueuedSynchronizer.java:1304) at java.util.concurrent.CountDownLatch.await(CountDownLatch.java:231) at kafka.server.KafkaServer.awaitShutdown(KafkaServer.scala:624) at kafka.server.KafkaServerStartable.awaitShutdown(KafkaServerStartable.scala:70) at kafka.Kafka$.main(Kafka.scala:68) at kafka.Kafka.main(Kafka.scala) "VM Thread" os_prio=0 tid=0x00007fb6c4336000 nid=0xbfe8 runnable "Gang worker#0 (Parallel GC Threads)" os_prio=0 tid=0x00007fb6c4025000 nid=0xbfb2 runnable "Gang worker#1 (Parallel GC Threads)" os_prio=0 tid=0x00007fb6c4027000 nid=0xbfb3 runnable "Gang worker#2 (Parallel GC Threads)" os_prio=0 tid=0x00007fb6c4029000 nid=0xbfb4 runnable "Gang worker#3 (Parallel GC Threads)" os_prio=0 tid=0x00007fb6c402a800 nid=0xbfb5 runnable "Gang worker#4 (Parallel GC Threads)" os_prio=0 tid=0x00007fb6c402c800 nid=0xbfb6 runnable "Gang worker#5 (Parallel GC Threads)" os_prio=0 tid=0x00007fb6c402e800 nid=0xbfb7 runnable "Gang worker#6 (Parallel GC Threads)" os_prio=0 tid=0x00007fb6c4030000 nid=0xbfb8 runnable "Gang worker#7 (Parallel GC Threads)" os_prio=0 tid=0x00007fb6c4032000 nid=0xbfb9 runnable "Gang worker#8 (Parallel GC Threads)" os_prio=0 tid=0x00007fb6c4033800 nid=0xbfba runnable "Gang worker#9 (Parallel GC Threads)" os_prio=0 tid=0x00007fb6c4035800 nid=0xbfbb runnable "Gang worker#10 (Parallel GC Threads)" os_prio=0 tid=0x00007fb6c4037800 nid=0xbfbc runnable "Gang worker#11 (Parallel GC Threads)" os_prio=0 tid=0x00007fb6c4039000 nid=0xbfbd runnable "Gang worker#12 (Parallel GC Threads)" os_prio=0 tid=0x00007fb6c403b000 nid=0xbfbe runnable "Gang worker#13 (Parallel GC Threads)" os_prio=0 tid=0x00007fb6c403d000 nid=0xbfbf runnable "Gang worker#14 (Parallel GC Threads)" os_prio=0 tid=0x00007fb6c403e800 nid=0xbfc0 runnable "Gang worker#15 (Parallel GC Threads)" os_prio=0 tid=0x00007fb6c4040800 nid=0xbfc1 runnable "Gang worker#16 (Parallel GC Threads)" os_prio=0 tid=0x00007fb6c4042800 nid=0xbfc2 runnable "Gang worker#17 (Parallel GC Threads)" os_prio=0 tid=0x00007fb6c4044000 nid=0xbfc3 runnable "Gang worker#18 (Parallel GC Threads)" os_prio=0 tid=0x00007fb6c4046000 nid=0xbfc4 runnable "Gang worker#19 (Parallel GC Threads)" os_prio=0 tid=0x00007fb6c4047800 nid=0xbfc5 runnable "Gang worker#20 (Parallel GC Threads)" os_prio=0 tid=0x00007fb6c4049800 nid=0xbfc6 runnable "Gang worker#21 (Parallel GC Threads)" os_prio=0 tid=0x00007fb6c404b800 nid=0xbfc7 runnable "Gang worker#22 (Parallel GC Threads)" os_prio=0 tid=0x00007fb6c404d000 nid=0xbfc8 runnable "G1 Main Concurrent Mark GC Thread" os_prio=0 tid=0x00007fb6c408f000 nid=0xbfe1 runnable "Gang worker#0 (G1 Parallel Marking Threads)" os_prio=0 tid=0x00007fb6c4090800 nid=0xbfe2 runnable "Gang worker#1 (G1 Parallel Marking Threads)" os_prio=0 tid=0x00007fb6c4092800 nid=0xbfe3 runnable "Gang worker#2 (G1 Parallel Marking Threads)" os_prio=0 tid=0x00007fb6c4094800 nid=0xbfe4 runnable "Gang worker#3 (G1 Parallel Marking Threads)" os_prio=0 tid=0x00007fb6c4096000 nid=0xbfe5 runnable "Gang worker#4 (G1 Parallel Marking Threads)" os_prio=0 tid=0x00007fb6c4098000 nid=0xbfe6 runnable "Gang worker#5 (G1 Parallel Marking Threads)" os_prio=0 tid=0x00007fb6c4099800 nid=0xbfe7 runnable "G1 Concurrent Refinement Thread#0" os_prio=0 tid=0x00007fb6c407b000 nid=0xbfe0 runnable "G1 Concurrent Refinement Thread#1" os_prio=0 tid=0x00007fb6c4079800 nid=0xbfdf runnable "G1 Concurrent Refinement Thread#2" os_prio=0 tid=0x00007fb6c4077800 nid=0xbfde runnable "G1 Concurrent Refinement Thread#3" os_prio=0 tid=0x00007fb6c4075800 nid=0xbfdd runnable "G1 Concurrent Refinement Thread#4" os_prio=0 tid=0x00007fb6c4074000 nid=0xbfdc runnable "G1 Concurrent Refinement Thread#5" os_prio=0 tid=0x00007fb6c4072000 nid=0xbfdb runnable "G1 Concurrent Refinement Thread#6" os_prio=0 tid=0x00007fb6c4070000 nid=0xbfda runnable "G1 Concurrent Refinement Thread#7" os_prio=0 tid=0x00007fb6c406e800 nid=0xbfd9 runnable "G1 Concurrent Refinement Thread#8" os_prio=0 tid=0x00007fb6c406c800 nid=0xbfd8 runnable "G1 Concurrent Refinement Thread#9" os_prio=0 tid=0x00007fb6c406a800 nid=0xbfd7 runnable "G1 Concurrent Refinement Thread#10" os_prio=0 tid=0x00007fb6c4068800 nid=0xbfd6 runnable "G1 Concurrent Refinement Thread#11" os_prio=0 tid=0x00007fb6c4067000 nid=0xbfd5 runnable "G1 Concurrent Refinement Thread#12" os_prio=0 tid=0x00007fb6c4065000 nid=0xbfd4 runnable "G1 Concurrent Refinement Thread#13" os_prio=0 tid=0x00007fb6c4063000 nid=0xbfd3 runnable "G1 Concurrent Refinement Thread#14" os_prio=0 tid=0x00007fb6c4061800 nid=0xbfd2 runnable "G1 Concurrent Refinement Thread#15" os_prio=0 tid=0x00007fb6c405f800 nid=0xbfd1 runnable "G1 Concurrent Refinement Thread#16" os_prio=0 tid=0x00007fb6c405d800 nid=0xbfd0 runnable "G1 Concurrent Refinement Thread#17" os_prio=0 tid=0x00007fb6c405c000 nid=0xbfcf runnable "G1 Concurrent Refinement Thread#18" os_prio=0 tid=0x00007fb6c405a000 nid=0xbfce runnable "G1 Concurrent Refinement Thread#19" os_prio=0 tid=0x00007fb6c4058000 nid=0xbfcd runnable "G1 Concurrent Refinement Thread#20" os_prio=0 tid=0x00007fb6c4056800 nid=0xbfcc runnable "G1 Concurrent Refinement Thread#21" os_prio=0 tid=0x00007fb6c4054800 nid=0xbfcb runnable "G1 Concurrent Refinement Thread#22" os_prio=0 tid=0x00007fb6c4052800 nid=0xbfca runnable "G1 Concurrent Refinement Thread#23" os_prio=0 tid=0x00007fb6c4051000 nid=0xbfc9 runnable "VM Periodic Task Thread" os_prio=0 tid=0x00007fb6c470d000 nid=0xc002 waiting on condition JNI global references: 2351