2019-09-17 11:24:06 Full thread dump OpenJDK 64-Bit Server VM (25.181-b02 mixed mode): "Attach Listener" #47059 daemon prio=9 os_prio=0 tid=0x00007f08e4006800 nid=0x3c03 waiting on condition [0x0000000000000000] java.lang.Thread.State: RUNNABLE Locked ownable synchronizers: - None "ReplicaFetcherThread-0-0" #46957 prio=5 os_prio=0 tid=0x00007f0864021000 nid=0x3b78 waiting on condition [0x00007f06b53f6000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00000000b8e8c328> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2163) at kafka.server.AbstractFetcherThread.$anonfun$maybeFetch$1(AbstractFetcherThread.scala:125) at kafka.server.AbstractFetcherThread$$Lambda$907/2136863218.apply(Unknown Source) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.maybeFetch(AbstractFetcherThread.scala:117) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:113) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "Controller-1-to-broker-0-send-thread" #46933 prio=5 os_prio=0 tid=0x00007f0854085000 nid=0x3b60 waiting on condition [0x00007f08484f1000] java.lang.Thread.State: WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00000000b93c3130> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) at kafka.controller.RequestSendThread.doWork(ControllerChannelManager.scala:223) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-0-2" #46607 prio=5 os_prio=0 tid=0x00007f0870028000 nid=0x3a1a runnable [0x00007f08483f0000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.FileDispatcherImpl.read0(Native Method) at sun.nio.ch.SocketDispatcher.read(SocketDispatcher.java:39) at sun.nio.ch.IOUtil.readIntoNativeBuffer(IOUtil.java:223) at sun.nio.ch.IOUtil.read(IOUtil.java:197) at sun.nio.ch.SocketChannelImpl.read(SocketChannelImpl.java:380) - locked <0x00000000ce2e7900> (a java.lang.Object) at org.apache.kafka.common.network.PlaintextTransportLayer.read(PlaintextTransportLayer.java:103) at org.apache.kafka.common.network.NetworkReceive.readFrom(NetworkReceive.java:117) at org.apache.kafka.common.network.KafkaChannel.receive(KafkaChannel.java:381) at org.apache.kafka.common.network.KafkaChannel.read(KafkaChannel.java:342) at org.apache.kafka.common.network.Selector.attemptRead(Selector.java:609) at org.apache.kafka.common.network.Selector.pollSelectionKeys(Selector.java:541) at org.apache.kafka.common.network.Selector.poll(Selector.java:467) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:535) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:96) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:99) at kafka.server.ReplicaFetcherThread.fetchFromLeader(ReplicaFetcherThread.scala:192) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:274) at kafka.server.AbstractFetcherThread.$anonfun$maybeFetch$3(AbstractFetcherThread.scala:132) at kafka.server.AbstractFetcherThread.$anonfun$maybeFetch$3$adapted(AbstractFetcherThread.scala:131) at kafka.server.AbstractFetcherThread$$Lambda$911/1478942588.apply(Unknown Source) at scala.Option.foreach(Option.scala:257) at kafka.server.AbstractFetcherThread.maybeFetch(AbstractFetcherThread.scala:131) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:113) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "Controller-1-to-broker-2-send-thread" #46606 prio=5 os_prio=0 tid=0x00007f0854167000 nid=0x3a19 waiting on condition [0x00007f084b1f2000] java.lang.Thread.State: WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00000000bc3e2878> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) at kafka.controller.RequestSendThread.doWork(ControllerChannelManager.scala:223) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "kafka-scheduler-1" #46604 daemon prio=5 os_prio=0 tid=0x00007f0854163800 nid=0x3a17 waiting on condition [0x00007f084b2f3000] java.lang.Thread.State: WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00000000bb97f578> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1081) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "Controller-1-to-broker-1-send-thread" #46603 prio=5 os_prio=0 tid=0x00007f08540a6000 nid=0x3a16 waiting on condition [0x00007f08cf170000] java.lang.Thread.State: WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00000000b8845000> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) at kafka.controller.RequestSendThread.doWork(ControllerChannelManager.scala:223) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "zk-session-expiry-handler0" #46599 daemon prio=5 os_prio=0 tid=0x00007f08b002d800 nid=0x3a12 waiting on condition [0x00007f0848f6b000] java.lang.Thread.State: WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00000000b8751488> (a java.util.concurrent.CountDownLatch$Sync) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer.parkAndCheckInterrupt(AbstractQueuedSynchronizer.java:836) at java.util.concurrent.locks.AbstractQueuedSynchronizer.doAcquireSharedInterruptibly(AbstractQueuedSynchronizer.java:997) at java.util.concurrent.locks.AbstractQueuedSynchronizer.acquireSharedInterruptibly(AbstractQueuedSynchronizer.java:1304) at java.util.concurrent.CountDownLatch.await(CountDownLatch.java:231) at kafka.controller.KafkaController$Expire.waitUntilProcessingStarted(KafkaController.scala:1533) at kafka.controller.KafkaController$$anon$7.beforeInitializingSession(KafkaController.scala:173) at kafka.zookeeper.ZooKeeperClient.callBeforeInitializingSession(ZooKeeperClient.scala:408) at kafka.zookeeper.ZooKeeperClient.$anonfun$reinitialize$1(ZooKeeperClient.scala:374) at kafka.zookeeper.ZooKeeperClient.$anonfun$reinitialize$1$adapted(ZooKeeperClient.scala:374) at kafka.zookeeper.ZooKeeperClient$$Lambda$1572/1731045326.apply(Unknown Source) at scala.collection.Iterator.foreach(Iterator.scala:937) at scala.collection.Iterator.foreach$(Iterator.scala:937) at scala.collection.AbstractIterator.foreach(Iterator.scala:1425) at scala.collection.MapLike$DefaultValuesIterable.foreach(MapLike.scala:209) at kafka.zookeeper.ZooKeeperClient.reinitialize(ZooKeeperClient.scala:374) at kafka.zookeeper.ZooKeeperClient.$anonfun$scheduleSessionExpiryHandler$1(ZooKeeperClient.scala:428) at kafka.zookeeper.ZooKeeperClient$$Lambda$1565/336537870.apply$mcV$sp(Unknown Source) at kafka.utils.KafkaScheduler.$anonfun$schedule$2(KafkaScheduler.scala:114) at kafka.utils.KafkaScheduler$$Lambda$198/2112233878.apply$mcV$sp(Unknown Source) at kafka.utils.CoreUtils$$anon$1.run(CoreUtils.scala:63) at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:511) at java.util.concurrent.FutureTask.run(FutureTask.java:266) at java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.access$201(ScheduledThreadPoolExecutor.java:180) at java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.run(ScheduledThreadPoolExecutor.java:293) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - <0x00000000bb96b090> (a java.util.concurrent.ThreadPoolExecutor$Worker) "executor-Produce" #246 prio=5 os_prio=0 tid=0x00007f088001c000 nid=0x379 waiting on condition [0x00007f081cdfa000] java.lang.Thread.State: WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00000000b7c74a88> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "executor-Heartbeat" #245 prio=5 os_prio=0 tid=0x00007f084c022000 nid=0x378 waiting on condition [0x00007f081cefb000] java.lang.Thread.State: WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00000000b7c7b350> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "executor-Rebalance" #239 prio=5 os_prio=0 tid=0x00007f0850013000 nid=0x372 waiting on condition [0x00007f08481ee000] java.lang.Thread.State: WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00000000b7bd61e8> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "RMI Scheduler(0)" #71 daemon prio=5 os_prio=0 tid=0x00007f08a800c000 nid=0x2ca waiting on condition [0x00007f081d3fe000] java.lang.Thread.State: WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00000000b8845038> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1081) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "executor-Fetch" #69 prio=5 os_prio=0 tid=0x00007f0874006000 nid=0x2c8 waiting on condition [0x00007f08482ef000] java.lang.Thread.State: WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00000000b7956f08> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "kafka-network-thread-1-ListenerName(PLAINTEXT)-PLAINTEXT-5" #63 prio=5 os_prio=0 tid=0x00007f095e5ca000 nid=0x2bf runnable [0x00007f084b4f5000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00000000b7c03e20> (a sun.nio.ch.Util$3) - locked <0x00000000b7c03e10> (a java.util.Collections$UnmodifiableSet) - locked <0x00000000b7c03e30> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:752) at org.apache.kafka.common.network.Selector.poll(Selector.java:451) at kafka.network.Processor.poll(SocketServer.scala:689) at kafka.network.Processor.run(SocketServer.scala:594) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "kafka-network-thread-1-ListenerName(PLAINTEXT)-PLAINTEXT-4" #62 prio=5 os_prio=0 tid=0x00007f095e5c8800 nid=0x2be runnable [0x00007f084b5f6000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00000000b7c03fc0> (a sun.nio.ch.Util$3) - locked <0x00000000b7c03fb0> (a java.util.Collections$UnmodifiableSet) - locked <0x00000000b7c03fd0> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:752) at org.apache.kafka.common.network.Selector.poll(Selector.java:451) at kafka.network.Processor.poll(SocketServer.scala:689) at kafka.network.Processor.run(SocketServer.scala:594) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "kafka-network-thread-1-ListenerName(PLAINTEXT)-PLAINTEXT-3" #61 prio=5 os_prio=0 tid=0x00007f095e5c7000 nid=0x2bd runnable [0x00007f084b6f7000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00000000b7b81fd8> (a sun.nio.ch.Util$3) - locked <0x00000000b7b81fc8> (a java.util.Collections$UnmodifiableSet) - locked <0x00000000b7b81fe8> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:752) at org.apache.kafka.common.network.Selector.poll(Selector.java:451) at kafka.network.Processor.poll(SocketServer.scala:689) at kafka.network.Processor.run(SocketServer.scala:594) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "kafka-network-thread-1-ListenerName(PLAINTEXT)-PLAINTEXT-2" #60 prio=5 os_prio=0 tid=0x00007f095e5c5000 nid=0x2bc runnable [0x00007f084b7f8000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00000000b7c04160> (a sun.nio.ch.Util$3) - locked <0x00000000b7c04150> (a java.util.Collections$UnmodifiableSet) - locked <0x00000000b7c04170> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:752) at org.apache.kafka.common.network.Selector.poll(Selector.java:451) at kafka.network.Processor.poll(SocketServer.scala:689) at kafka.network.Processor.run(SocketServer.scala:594) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "kafka-network-thread-1-ListenerName(PLAINTEXT)-PLAINTEXT-1" #59 prio=5 os_prio=0 tid=0x00007f095e5c3800 nid=0x2bb runnable [0x00007f084b8f9000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00000000b7c04300> (a sun.nio.ch.Util$3) - locked <0x00000000b7c042f0> (a java.util.Collections$UnmodifiableSet) - locked <0x00000000b7c04310> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:752) at org.apache.kafka.common.network.Selector.poll(Selector.java:451) at kafka.network.Processor.poll(SocketServer.scala:689) at kafka.network.Processor.run(SocketServer.scala:594) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "kafka-network-thread-1-ListenerName(PLAINTEXT)-PLAINTEXT-0" #58 prio=5 os_prio=0 tid=0x00007f095e5c2000 nid=0x2ba runnable [0x00007f084b9fa000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00000000b7c044a0> (a sun.nio.ch.Util$3) - locked <0x00000000b7c04490> (a java.util.Collections$UnmodifiableSet) - locked <0x00000000b7c044b0> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:752) at org.apache.kafka.common.network.Selector.poll(Selector.java:451) at kafka.network.Processor.poll(SocketServer.scala:689) at kafka.network.Processor.run(SocketServer.scala:594) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "/config/changes-event-process-thread" #57 prio=5 os_prio=0 tid=0x00007f095e5bb000 nid=0x2b9 waiting on condition [0x00007f084bafb000] java.lang.Thread.State: WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00000000b8a98808> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) at kafka.common.ZkNodeChangeNotificationListener$ChangeEventProcessThread.doWork(ZkNodeChangeNotificationListener.scala:145) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "kafka-request-handler-7" #56 daemon prio=5 os_prio=0 tid=0x00007f095e5ac000 nid=0x2b8 runnable [0x00007f084bdfc000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00000000b7b82120> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.ArrayBlockingQueue.poll(ArrayBlockingQueue.java:418) at kafka.network.RequestChannel.receiveRequest(RequestChannel.scala:342) at kafka.server.KafkaRequestHandler.run(KafkaRequestHandler.scala:54) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "kafka-request-handler-6" #55 daemon prio=5 os_prio=0 tid=0x00007f095e5aa800 nid=0x2b7 runnable [0x00007f084befd000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00000000b7b82120> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.ArrayBlockingQueue.poll(ArrayBlockingQueue.java:418) at kafka.network.RequestChannel.receiveRequest(RequestChannel.scala:342) at kafka.server.KafkaRequestHandler.run(KafkaRequestHandler.scala:54) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "kafka-request-handler-5" #54 daemon prio=5 os_prio=0 tid=0x00007f095e5a8800 nid=0x2b6 runnable [0x00007f084bffe000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00000000b7b82120> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.ArrayBlockingQueue.poll(ArrayBlockingQueue.java:418) at kafka.network.RequestChannel.receiveRequest(RequestChannel.scala:342) at kafka.server.KafkaRequestHandler.run(KafkaRequestHandler.scala:54) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "kafka-request-handler-4" #53 daemon prio=5 os_prio=0 tid=0x00007f095e5a7000 nid=0x2b5 runnable [0x00007f08cc130000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00000000b7b82120> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.ArrayBlockingQueue.poll(ArrayBlockingQueue.java:418) at kafka.network.RequestChannel.receiveRequest(RequestChannel.scala:342) at kafka.server.KafkaRequestHandler.run(KafkaRequestHandler.scala:54) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "kafka-request-handler-3" #52 daemon prio=5 os_prio=0 tid=0x00007f095e5a5000 nid=0x2b4 runnable [0x00007f08cc231000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00000000b7b82120> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.ArrayBlockingQueue.poll(ArrayBlockingQueue.java:418) at kafka.network.RequestChannel.receiveRequest(RequestChannel.scala:342) at kafka.server.KafkaRequestHandler.run(KafkaRequestHandler.scala:54) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "kafka-request-handler-2" #51 daemon prio=5 os_prio=0 tid=0x00007f095e5a3800 nid=0x2b3 runnable [0x00007f08cc332000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00000000b7b82120> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.ArrayBlockingQueue.poll(ArrayBlockingQueue.java:418) at kafka.network.RequestChannel.receiveRequest(RequestChannel.scala:342) at kafka.server.KafkaRequestHandler.run(KafkaRequestHandler.scala:54) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "kafka-request-handler-1" #50 daemon prio=5 os_prio=0 tid=0x00007f095e5a0800 nid=0x2b2 runnable [0x00007f08cc433000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00000000b7b82120> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.ArrayBlockingQueue.poll(ArrayBlockingQueue.java:418) at kafka.network.RequestChannel.receiveRequest(RequestChannel.scala:342) at kafka.server.KafkaRequestHandler.run(KafkaRequestHandler.scala:54) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "kafka-request-handler-0" #49 daemon prio=5 os_prio=0 tid=0x00007f095e59f800 nid=0x2b1 runnable [0x00007f08cc534000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00000000b7b82120> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.ArrayBlockingQueue.poll(ArrayBlockingQueue.java:418) at kafka.network.RequestChannel.receiveRequest(RequestChannel.scala:342) at kafka.server.KafkaRequestHandler.run(KafkaRequestHandler.scala:54) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "TxnMarkerSenderThread-1" #46 prio=5 os_prio=0 tid=0x00007f095e58d800 nid=0x2af runnable [0x00007f08cc736000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00000000b7c04800> (a sun.nio.ch.Util$3) - locked <0x00000000b7c047f0> (a java.util.Collections$UnmodifiableSet) - locked <0x00000000b7c04810> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:752) at org.apache.kafka.common.network.Selector.poll(Selector.java:451) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:535) at kafka.common.InterBrokerSendThread.doWork(InterBrokerSendThread.scala:66) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "transaction-log-manager-0" #47 daemon prio=5 os_prio=0 tid=0x00007f095e58b800 nid=0x2ae waiting on condition [0x00007f08cc837000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00000000b7b82518> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1093) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "group-metadata-manager-0" #44 daemon prio=5 os_prio=0 tid=0x00007f095e526000 nid=0x2ad waiting on condition [0x00007f08ccb38000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00000000b7c04aa8> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1093) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "ExpirationReaper-1-Rebalance" #43 prio=5 os_prio=0 tid=0x00007f095e517800 nid=0x2ac runnable [0x00007f08ccc39000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00000000b7b82778> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.DelayQueue.poll(DelayQueue.java:268) at kafka.utils.timer.SystemTimer.advanceClock(Timer.scala:105) at kafka.server.DelayedOperationPurgatory.advanceClock(DelayedOperation.scala:416) at kafka.server.DelayedOperationPurgatory$ExpiredOperationReaper.doWork(DelayedOperation.scala:440) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ExpirationReaper-1-Heartbeat" #42 prio=5 os_prio=0 tid=0x00007f095e515800 nid=0x2ab runnable [0x00007f08ccd3a000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00000000b7c04d08> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.DelayQueue.poll(DelayQueue.java:259) at kafka.utils.timer.SystemTimer.advanceClock(Timer.scala:105) at kafka.server.DelayedOperationPurgatory.advanceClock(DelayedOperation.scala:416) at kafka.server.DelayedOperationPurgatory$ExpiredOperationReaper.doWork(DelayedOperation.scala:440) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ExpirationReaper-1-topic" #41 prio=5 os_prio=0 tid=0x00007f095e50b800 nid=0x2aa runnable [0x00007f08cce3b000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00000000b7b829c8> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.DelayQueue.poll(DelayQueue.java:259) at kafka.utils.timer.SystemTimer.advanceClock(Timer.scala:105) at kafka.server.DelayedOperationPurgatory.advanceClock(DelayedOperation.scala:416) at kafka.server.DelayedOperationPurgatory$ExpiredOperationReaper.doWork(DelayedOperation.scala:440) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "controller-event-thread" #40 prio=5 os_prio=0 tid=0x00007f095e504000 nid=0x2a9 waiting on condition [0x00007f08ccf3c000] java.lang.Thread.State: WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00000000b8b018f8> (a java.util.concurrent.CountDownLatch$Sync) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer.parkAndCheckInterrupt(AbstractQueuedSynchronizer.java:836) at java.util.concurrent.locks.AbstractQueuedSynchronizer.doAcquireSharedInterruptibly(AbstractQueuedSynchronizer.java:997) at java.util.concurrent.locks.AbstractQueuedSynchronizer.acquireSharedInterruptibly(AbstractQueuedSynchronizer.java:1304) at java.util.concurrent.CountDownLatch.await(CountDownLatch.java:231) at kafka.zookeeper.ZooKeeperClient.handleRequests(ZooKeeperClient.scala:157) at kafka.zk.KafkaZkClient.retryRequestsUntilConnected(KafkaZkClient.scala:1596) at kafka.zk.KafkaZkClient.deleteIsrChangeNotifications(KafkaZkClient.scala:956) at kafka.controller.KafkaController$IsrChangeNotification$.process(KafkaController.scala:1464) at kafka.controller.ControllerEventManager$ControllerEventThread.$anonfun$doWork$1(ControllerEventManager.scala:89) at kafka.controller.ControllerEventManager$ControllerEventThread$$Lambda$361/957068261.apply$mcV$sp(Unknown Source) at scala.runtime.java8.JFunction0$mcV$sp.apply(JFunction0$mcV$sp.java:12) at kafka.metrics.KafkaTimer.time(KafkaTimer.scala:31) at kafka.controller.ControllerEventManager$ControllerEventThread.doWork(ControllerEventManager.scala:89) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "kafka-scheduler-9" #39 daemon prio=5 os_prio=0 tid=0x00007f0884003000 nid=0x2a8 waiting on condition [0x00007f08cd03d000] java.lang.Thread.State: WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00000000b7b82c00> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1088) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "LogDirFailureHandler" #38 prio=5 os_prio=0 tid=0x00007f095e495800 nid=0x2a7 waiting on condition [0x00007f08cd13e000] java.lang.Thread.State: WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00000000b7c05158> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) at java.util.concurrent.ArrayBlockingQueue.take(ArrayBlockingQueue.java:403) at kafka.server.LogDirFailureChannel.takeNextOfflineLogDir(LogDirFailureChannel.scala:56) at kafka.server.ReplicaManager$LogDirFailureHandler.doWork(ReplicaManager.scala:200) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "kafka-scheduler-8" #37 daemon prio=5 os_prio=0 tid=0x00007f088c002000 nid=0x2a6 waiting on condition [0x00007f08cd23f000] java.lang.Thread.State: WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00000000b7b82c00> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1088) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "kafka-scheduler-7" #36 daemon prio=5 os_prio=0 tid=0x00007f095e493000 nid=0x2a5 waiting on condition [0x00007f08cd340000] java.lang.Thread.State: WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00000000b7b82c00> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1088) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "kafka-scheduler-6" #35 daemon prio=5 os_prio=0 tid=0x00007f095e491000 nid=0x2a4 waiting on condition [0x00007f08cd441000] java.lang.Thread.State: WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00000000b7b82c00> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1088) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "kafka-scheduler-5" #34 daemon prio=5 os_prio=0 tid=0x00007f095e498000 nid=0x2a3 waiting on condition [0x00007f08cd542000] java.lang.Thread.State: WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00000000b7b82c00> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1088) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "ExpirationReaper-1-DeleteRecords" #33 prio=5 os_prio=0 tid=0x00007f095e462800 nid=0x2a2 runnable [0x00007f08cd643000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00000000b7c05538> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.DelayQueue.poll(DelayQueue.java:259) at kafka.utils.timer.SystemTimer.advanceClock(Timer.scala:105) at kafka.server.DelayedOperationPurgatory.advanceClock(DelayedOperation.scala:416) at kafka.server.DelayedOperationPurgatory$ExpiredOperationReaper.doWork(DelayedOperation.scala:440) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ExpirationReaper-1-Fetch" #32 prio=5 os_prio=0 tid=0x00007f095e460800 nid=0x2a1 runnable [0x00007f08cd744000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00000000b7d37188> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.DelayQueue.poll(DelayQueue.java:273) at kafka.utils.timer.SystemTimer.advanceClock(Timer.scala:105) at kafka.server.DelayedOperationPurgatory.advanceClock(DelayedOperation.scala:416) at kafka.server.DelayedOperationPurgatory$ExpiredOperationReaper.doWork(DelayedOperation.scala:440) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ExpirationReaper-1-Produce" #31 prio=5 os_prio=0 tid=0x00007f095e45f000 nid=0x2a0 runnable [0x00007f08cd845000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00000000b7c05788> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.DelayQueue.poll(DelayQueue.java:259) at kafka.utils.timer.SystemTimer.advanceClock(Timer.scala:105) at kafka.server.DelayedOperationPurgatory.advanceClock(DelayedOperation.scala:416) at kafka.server.DelayedOperationPurgatory$ExpiredOperationReaper.doWork(DelayedOperation.scala:440) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "kafka-socket-acceptor-ListenerName(PLAINTEXT)-PLAINTEXT-9092" #30 prio=5 os_prio=0 tid=0x00007f095e3b5000 nid=0x29f runnable [0x00007f08cdb46000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00000000b7c05a18> (a sun.nio.ch.Util$3) - locked <0x00000000b7c05a08> (a java.util.Collections$UnmodifiableSet) - locked <0x00000000b7c05a28> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at kafka.network.Acceptor.run(SocketServer.scala:390) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "kafka-log-cleaner-thread-0" #29 prio=5 os_prio=0 tid=0x00007f095e2f2000 nid=0x29e waiting on condition [0x00007f08cde47000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00000000b7c05c60> (a java.util.concurrent.CountDownLatch$Sync) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer.doAcquireSharedNanos(AbstractQueuedSynchronizer.java:1037) at java.util.concurrent.locks.AbstractQueuedSynchronizer.tryAcquireSharedNanos(AbstractQueuedSynchronizer.java:1328) at java.util.concurrent.CountDownLatch.await(CountDownLatch.java:277) at kafka.utils.ShutdownableThread.pause(ShutdownableThread.scala:69) at kafka.log.LogCleaner$CleanerThread.doWork(LogCleaner.scala:302) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "kafka-scheduler-4" #28 daemon prio=5 os_prio=0 tid=0x00007f095e2e2000 nid=0x29d waiting on condition [0x00007f08cdf48000] java.lang.Thread.State: WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00000000b7b82c00> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1088) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "kafka-scheduler-3" #27 daemon prio=5 os_prio=0 tid=0x00007f095e2e0000 nid=0x29c waiting on condition [0x00007f08ce049000] java.lang.Thread.State: WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00000000b7b82c00> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1088) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "kafka-scheduler-2" #26 daemon prio=5 os_prio=0 tid=0x00007f095e2de000 nid=0x29b waiting on condition [0x00007f08ce14a000] java.lang.Thread.State: WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00000000b7b82c00> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1088) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "kafka-scheduler-1" #25 daemon prio=5 os_prio=0 tid=0x00007f095e2dc000 nid=0x29a waiting on condition [0x00007f08ce24b000] java.lang.Thread.State: WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00000000b7b82c00> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1088) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "kafka-scheduler-0" #24 daemon prio=5 os_prio=0 tid=0x00007f095e2d9800 nid=0x299 waiting on condition [0x00007f08ce34c000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00000000b7b82c00> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1093) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "ThrottledChannelReaper-Request" #23 prio=5 os_prio=0 tid=0x00007f095e2a0800 nid=0x298 runnable [0x00007f08ce44d000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00000000b7c061a8> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.DelayQueue.poll(DelayQueue.java:259) at kafka.server.ClientQuotaManager$ThrottledChannelReaper.doWork(ClientQuotaManager.scala:196) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ThrottledChannelReaper-Produce" #22 prio=5 os_prio=0 tid=0x00007f095e29e000 nid=0x297 runnable [0x00007f08ce54e000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00000000b7d3e658> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.DelayQueue.poll(DelayQueue.java:259) at kafka.server.ClientQuotaManager$ThrottledChannelReaper.doWork(ClientQuotaManager.scala:196) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ThrottledChannelReaper-Fetch" #21 prio=5 os_prio=0 tid=0x00007f095e299800 nid=0x296 runnable [0x00007f08ce64f000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00000000b7d3de20> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.DelayQueue.poll(DelayQueue.java:259) at kafka.server.ClientQuotaManager$ThrottledChannelReaper.doWork(ClientQuotaManager.scala:196) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "SensorExpiryThread" #20 daemon prio=5 os_prio=0 tid=0x00007f095e27f800 nid=0x295 waiting on condition [0x00007f08ce750000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00000000b7b83560> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1093) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "metrics-meter-tick-thread-2" #17 daemon prio=5 os_prio=0 tid=0x00007f095e0cb000 nid=0x292 waiting on condition [0x00007f08cf471000] java.lang.Thread.State: WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00000000b7fbbe60> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1088) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "metrics-meter-tick-thread-1" #16 daemon prio=5 os_prio=0 tid=0x00007f095e0c8800 nid=0x291 runnable [0x00007f08cf572000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00000000b7fbbe60> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1093) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "RMI TCP Accept-0" #13 daemon prio=5 os_prio=0 tid=0x00007f095c8e3800 nid=0x28f runnable [0x00007f08cfd74000] java.lang.Thread.State: RUNNABLE at java.net.PlainSocketImpl.socketAccept(Native Method) at java.net.AbstractPlainSocketImpl.accept(AbstractPlainSocketImpl.java:409) at java.net.ServerSocket.implAccept(ServerSocket.java:545) at java.net.ServerSocket.accept(ServerSocket.java:513) at sun.management.jmxremote.LocalRMIServerSocketFactory$1.accept(LocalRMIServerSocketFactory.java:52) at sun.rmi.transport.tcp.TCPTransport$AcceptLoop.executeAcceptLoop(TCPTransport.java:405) at sun.rmi.transport.tcp.TCPTransport$AcceptLoop.run(TCPTransport.java:377) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "RMI TCP Accept-4535" #12 daemon prio=5 os_prio=0 tid=0x00007f095c8c1800 nid=0x28e runnable [0x00007f08cfe75000] java.lang.Thread.State: RUNNABLE at java.net.PlainSocketImpl.socketAccept(Native Method) at java.net.AbstractPlainSocketImpl.accept(AbstractPlainSocketImpl.java:409) at java.net.ServerSocket.implAccept(ServerSocket.java:545) at java.net.ServerSocket.accept(ServerSocket.java:513) at sun.rmi.transport.tcp.TCPTransport$AcceptLoop.executeAcceptLoop(TCPTransport.java:405) at sun.rmi.transport.tcp.TCPTransport$AcceptLoop.run(TCPTransport.java:377) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "Service Thread" #10 daemon prio=9 os_prio=0 tid=0x00007f095c568000 nid=0x28d runnable [0x0000000000000000] java.lang.Thread.State: RUNNABLE Locked ownable synchronizers: - None "C1 CompilerThread3" #9 daemon prio=9 os_prio=0 tid=0x00007f095c555000 nid=0x28c waiting on condition [0x0000000000000000] java.lang.Thread.State: RUNNABLE Locked ownable synchronizers: - None "C2 CompilerThread2" #8 daemon prio=9 os_prio=0 tid=0x00007f095c553000 nid=0x28b waiting on condition [0x0000000000000000] java.lang.Thread.State: RUNNABLE Locked ownable synchronizers: - None "C2 CompilerThread1" #7 daemon prio=9 os_prio=0 tid=0x00007f095c551000 nid=0x28a waiting on condition [0x0000000000000000] java.lang.Thread.State: RUNNABLE Locked ownable synchronizers: - None "C2 CompilerThread0" #6 daemon prio=9 os_prio=0 tid=0x00007f095c54e800 nid=0x289 waiting on condition [0x0000000000000000] java.lang.Thread.State: RUNNABLE Locked ownable synchronizers: - None "Signal Dispatcher" #5 daemon prio=9 os_prio=0 tid=0x00007f095c54a800 nid=0x288 runnable [0x0000000000000000] java.lang.Thread.State: RUNNABLE Locked ownable synchronizers: - None "Surrogate Locker Thread (Concurrent GC)" #4 daemon prio=9 os_prio=0 tid=0x00007f095c53a800 nid=0x287 waiting on condition [0x0000000000000000] java.lang.Thread.State: RUNNABLE Locked ownable synchronizers: - None "Finalizer" #3 daemon prio=8 os_prio=0 tid=0x00007f095c51a000 nid=0x286 in Object.wait() [0x00007f0915165000] java.lang.Thread.State: WAITING (on object monitor) at java.lang.Object.wait(Native Method) at java.lang.ref.ReferenceQueue.remove(ReferenceQueue.java:144) - locked <0x00000000b7f7cf48> (a java.lang.ref.ReferenceQueue$Lock) at java.lang.ref.ReferenceQueue.remove(ReferenceQueue.java:165) at java.lang.ref.Finalizer$FinalizerThread.run(Finalizer.java:216) Locked ownable synchronizers: - None "Reference Handler" #2 daemon prio=10 os_prio=0 tid=0x00007f095c515800 nid=0x285 in Object.wait() [0x00007f0915266000] java.lang.Thread.State: WAITING (on object monitor) at java.lang.Object.wait(Native Method) at java.lang.Object.wait(Object.java:502) at java.lang.ref.Reference.tryHandlePending(Reference.java:191) - locked <0x00000000b7f7cc58> (a java.lang.ref.Reference$Lock) at java.lang.ref.Reference$ReferenceHandler.run(Reference.java:153) Locked ownable synchronizers: - None "main" #1 prio=5 os_prio=0 tid=0x00007f095c010000 nid=0x26f waiting on condition [0x00007f09631a7000] java.lang.Thread.State: WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00000000b8a899c8> (a java.util.concurrent.CountDownLatch$Sync) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer.parkAndCheckInterrupt(AbstractQueuedSynchronizer.java:836) at java.util.concurrent.locks.AbstractQueuedSynchronizer.doAcquireSharedInterruptibly(AbstractQueuedSynchronizer.java:997) at java.util.concurrent.locks.AbstractQueuedSynchronizer.acquireSharedInterruptibly(AbstractQueuedSynchronizer.java:1304) at java.util.concurrent.CountDownLatch.await(CountDownLatch.java:231) at kafka.server.KafkaServer.awaitShutdown(KafkaServer.scala:644) at kafka.server.KafkaServerStartable.awaitShutdown(KafkaServerStartable.scala:65) at kafka.Kafka$.main(Kafka.scala:76) at kafka.Kafka.main(Kafka.scala) Locked ownable synchronizers: - None "VM Thread" os_prio=0 tid=0x00007f095c50d800 nid=0x284 runnable "Gang worker#0 (Parallel GC Threads)" os_prio=0 tid=0x00007f095c024800 nid=0x270 runnable "Gang worker#1 (Parallel GC Threads)" os_prio=0 tid=0x00007f095c026800 nid=0x271 runnable "Gang worker#2 (Parallel GC Threads)" os_prio=0 tid=0x00007f095c028000 nid=0x272 runnable "Gang worker#3 (Parallel GC Threads)" os_prio=0 tid=0x00007f095c02a000 nid=0x273 runnable "Gang worker#4 (Parallel GC Threads)" os_prio=0 tid=0x00007f095c02b800 nid=0x274 runnable "Gang worker#5 (Parallel GC Threads)" os_prio=0 tid=0x00007f095c02d800 nid=0x275 runnable "Gang worker#6 (Parallel GC Threads)" os_prio=0 tid=0x00007f095c02f000 nid=0x276 runnable "Gang worker#7 (Parallel GC Threads)" os_prio=0 tid=0x00007f095c031000 nid=0x277 runnable "G1 Main Concurrent Mark GC Thread" os_prio=0 tid=0x00007f095c055800 nid=0x281 runnable "Gang worker#0 (G1 Parallel Marking Threads)" os_prio=0 tid=0x00007f095c057000 nid=0x282 runnable "Gang worker#1 (G1 Parallel Marking Threads)" os_prio=0 tid=0x00007f095c059000 nid=0x283 runnable "G1 Concurrent Refinement Thread#0" os_prio=0 tid=0x00007f095c041800 nid=0x280 runnable "G1 Concurrent Refinement Thread#1" os_prio=0 tid=0x00007f095c03f800 nid=0x27f runnable "G1 Concurrent Refinement Thread#2" os_prio=0 tid=0x00007f095c03e000 nid=0x27e runnable "G1 Concurrent Refinement Thread#3" os_prio=0 tid=0x00007f095c03c000 nid=0x27d runnable "G1 Concurrent Refinement Thread#4" os_prio=0 tid=0x00007f095c03a800 nid=0x27c runnable "G1 Concurrent Refinement Thread#5" os_prio=0 tid=0x00007f095c038800 nid=0x27b runnable "G1 Concurrent Refinement Thread#6" os_prio=0 tid=0x00007f095c037000 nid=0x27a runnable "G1 Concurrent Refinement Thread#7" os_prio=0 tid=0x00007f095c035000 nid=0x279 runnable "G1 Concurrent Refinement Thread#8" os_prio=0 tid=0x00007f095c033800 nid=0x278 runnable "VM Periodic Task Thread" os_prio=0 tid=0x00007f095c8e5800 nid=0x290 waiting on condition JNI global references: 3982