2018-02-09 12:34:47 Full thread dump Java HotSpot(TM) 64-Bit Server VM (25.91-b14 mixed mode): "Attach Listener" #62 daemon prio=9 os_prio=31 tid=0x00007f83e3abb000 nid=0x7b0b waiting on condition [0x0000000000000000] java.lang.Thread.State: RUNNABLE Locked ownable synchronizers: - None "kafka-shutdown-hook" #14 prio=5 os_prio=31 tid=0x00007f83e29b1000 nid=0x560f waiting on condition [0x0000700005208000] java.lang.Thread.State: WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x0000000780054860> (a java.util.concurrent.locks.ReentrantReadWriteLock$NonfairSync) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer.parkAndCheckInterrupt(AbstractQueuedSynchronizer.java:836) at java.util.concurrent.locks.AbstractQueuedSynchronizer.doAcquireShared(AbstractQueuedSynchronizer.java:967) at java.util.concurrent.locks.AbstractQueuedSynchronizer.acquireShared(AbstractQueuedSynchronizer.java:1283) at java.util.concurrent.locks.ReentrantReadWriteLock$ReadLock.lock(ReentrantReadWriteLock.java:727) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:248) at kafka.utils.CoreUtils$.inReadLock(CoreUtils.scala:256) at kafka.zookeeper.ZooKeeperClient$$anonfun$handleRequests$1.apply(ZooKeeperClient.scala:135) at kafka.zookeeper.ZooKeeperClient$$anonfun$handleRequests$1.apply(ZooKeeperClient.scala:132) at scala.collection.mutable.ResizableArray$class.foreach(ResizableArray.scala:59) at scala.collection.mutable.ArrayBuffer.foreach(ArrayBuffer.scala:48) at kafka.zookeeper.ZooKeeperClient.handleRequests(ZooKeeperClient.scala:132) at kafka.zk.KafkaZkClient.retryRequestsUntilConnected(KafkaZkClient.scala:1439) at kafka.zk.KafkaZkClient.kafka$zk$KafkaZkClient$$retryRequestUntilConnected(KafkaZkClient.scala:1432) at kafka.zk.KafkaZkClient.getControllerId(KafkaZkClient.scala:862) at kafka.server.KafkaServer.doControlledShutdown$1(KafkaServer.scala:458) at kafka.server.KafkaServer.kafka$server$KafkaServer$$controlledShutdown(KafkaServer.scala:534) at kafka.server.KafkaServer$$anonfun$shutdown$1.apply$mcV$sp(KafkaServer.scala:556) at kafka.utils.CoreUtils$.swallow(CoreUtils.scala:85) at kafka.server.KafkaServer.shutdown(KafkaServer.scala:556) at kafka.server.KafkaServerStartable.shutdown(KafkaServerStartable.scala:48) at kafka.Kafka$$anon$1.run(Kafka.scala:89) Locked ownable synchronizers: - None "SIGINT handler" #61 daemon prio=9 os_prio=31 tid=0x00007f83e429a800 nid=0x1317 in Object.wait() [0x0000700001a61000] java.lang.Thread.State: WAITING (on object monitor) at java.lang.Object.wait(Native Method) - waiting on <0x00000007887b52b8> (a kafka.Kafka$$anon$1) at java.lang.Thread.join(Thread.java:1245) - locked <0x00000007887b52b8> (a kafka.Kafka$$anon$1) at java.lang.Thread.join(Thread.java:1319) at java.lang.ApplicationShutdownHooks.runHooks(ApplicationShutdownHooks.java:106) at java.lang.ApplicationShutdownHooks$1.run(ApplicationShutdownHooks.java:46) at java.lang.Shutdown.runHooks(Shutdown.java:123) at java.lang.Shutdown.sequence(Shutdown.java:167) at java.lang.Shutdown.exit(Shutdown.java:212) - locked <0x0000000788799218> (a java.lang.Class for java.lang.Shutdown) at java.lang.Terminator$1.handle(Terminator.java:52) at kafka.Kafka$$anon$2$$anonfun$handle$2.apply(Kafka.scala:63) at kafka.Kafka$$anon$2$$anonfun$handle$2.apply(Kafka.scala:63) at scala.Option.foreach(Option.scala:257) at kafka.Kafka$$anon$2.handle(Kafka.scala:63) at sun.misc.Signal$1.run(Signal.java:212) at java.lang.Thread.run(Thread.java:745) Locked ownable synchronizers: - None "kafka-scheduler-0" #60 daemon prio=5 os_prio=31 tid=0x00007f83e2978800 nid=0xcf03 waiting on condition [0x0000700005106000] java.lang.Thread.State: WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00000007bcb02700> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1081) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1067) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) at java.lang.Thread.run(Thread.java:745) Locked ownable synchronizers: - None "/config/changes-event-process-thread" #59 prio=5 os_prio=31 tid=0x00007f83e583e000 nid=0xcd03 waiting on condition [0x0000700005003000] java.lang.Thread.State: WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00000007bcc57198> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) at kafka.common.ZkNodeChangeNotificationListener$ChangeEventProcessThread.doWork(ZkNodeChangeNotificationListener.scala:135) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "kafka-request-handler-7" #58 daemon prio=5 os_prio=31 tid=0x00007f83e5025000 nid=0xcb03 waiting on condition [0x0000700004f00000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00000007bcb56dc8> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.ArrayBlockingQueue.poll(ArrayBlockingQueue.java:418) at kafka.network.RequestChannel.receiveRequest(RequestChannel.scala:309) at kafka.server.KafkaRequestHandler.run(KafkaRequestHandler.scala:54) at java.lang.Thread.run(Thread.java:745) Locked ownable synchronizers: - None "kafka-request-handler-6" #57 daemon prio=5 os_prio=31 tid=0x00007f83e5024800 nid=0xc903 waiting on condition [0x0000700004dfd000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00000007bcb56dc8> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.ArrayBlockingQueue.poll(ArrayBlockingQueue.java:418) at kafka.network.RequestChannel.receiveRequest(RequestChannel.scala:309) at kafka.server.KafkaRequestHandler.run(KafkaRequestHandler.scala:54) at java.lang.Thread.run(Thread.java:745) Locked ownable synchronizers: - None "kafka-request-handler-5" #56 daemon prio=5 os_prio=31 tid=0x00007f83e5023800 nid=0xc703 waiting on condition [0x0000700004cfa000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00000007bcb56dc8> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.ArrayBlockingQueue.poll(ArrayBlockingQueue.java:418) at kafka.network.RequestChannel.receiveRequest(RequestChannel.scala:309) at kafka.server.KafkaRequestHandler.run(KafkaRequestHandler.scala:54) at java.lang.Thread.run(Thread.java:745) Locked ownable synchronizers: - None "kafka-request-handler-4" #55 daemon prio=5 os_prio=31 tid=0x00007f83e5012000 nid=0xc503 waiting on condition [0x0000700004bf7000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00000007bcb56dc8> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.ArrayBlockingQueue.poll(ArrayBlockingQueue.java:418) at kafka.network.RequestChannel.receiveRequest(RequestChannel.scala:309) at kafka.server.KafkaRequestHandler.run(KafkaRequestHandler.scala:54) at java.lang.Thread.run(Thread.java:745) Locked ownable synchronizers: - None "kafka-request-handler-3" #54 daemon prio=5 os_prio=31 tid=0x00007f83e6072800 nid=0xc303 waiting on condition [0x0000700004af4000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00000007bcb56dc8> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.ArrayBlockingQueue.poll(ArrayBlockingQueue.java:418) at kafka.network.RequestChannel.receiveRequest(RequestChannel.scala:309) at kafka.server.KafkaRequestHandler.run(KafkaRequestHandler.scala:54) at java.lang.Thread.run(Thread.java:745) Locked ownable synchronizers: - None "kafka-request-handler-2" #53 daemon prio=5 os_prio=31 tid=0x00007f83e42a6000 nid=0xc103 waiting on condition [0x00007000049f1000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00000007bcb56dc8> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.ArrayBlockingQueue.poll(ArrayBlockingQueue.java:418) at kafka.network.RequestChannel.receiveRequest(RequestChannel.scala:309) at kafka.server.KafkaRequestHandler.run(KafkaRequestHandler.scala:54) at java.lang.Thread.run(Thread.java:745) Locked ownable synchronizers: - None "kafka-request-handler-1" #52 daemon prio=5 os_prio=31 tid=0x00007f83e583c800 nid=0xbf03 waiting on condition [0x00007000048ee000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00000007bcb56dc8> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.ArrayBlockingQueue.poll(ArrayBlockingQueue.java:418) at kafka.network.RequestChannel.receiveRequest(RequestChannel.scala:309) at kafka.server.KafkaRequestHandler.run(KafkaRequestHandler.scala:54) at java.lang.Thread.run(Thread.java:745) Locked ownable synchronizers: - None "kafka-request-handler-0" #51 daemon prio=5 os_prio=31 tid=0x00007f83e3366000 nid=0xbd03 waiting on condition [0x00007000047eb000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00000007bcb56dc8> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.ArrayBlockingQueue.poll(ArrayBlockingQueue.java:418) at kafka.network.RequestChannel.receiveRequest(RequestChannel.scala:309) at kafka.server.KafkaRequestHandler.run(KafkaRequestHandler.scala:54) at java.lang.Thread.run(Thread.java:745) Locked ownable synchronizers: - None "Controller-0-to-broker-0-send-thread" #49 prio=5 os_prio=31 tid=0x00007f83e3d3f800 nid=0xbb03 waiting on condition [0x00007000046e8000] java.lang.Thread.State: WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00000007bcc57b38> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) at kafka.controller.RequestSendThread.doWork(ControllerChannelManager.scala:218) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "TxnMarkerSenderThread-0" #48 prio=5 os_prio=31 tid=0x00007f83e3d37000 nid=0xb903 runnable [0x00007000045e5000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.KQueueArrayWrapper.kevent0(Native Method) at sun.nio.ch.KQueueArrayWrapper.poll(KQueueArrayWrapper.java:198) at sun.nio.ch.KQueueSelectorImpl.doSelect(KQueueSelectorImpl.java:117) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00000007bccac330> (a sun.nio.ch.Util$2) - locked <0x00000007bccac340> (a java.util.Collections$UnmodifiableSet) - locked <0x00000007bccac2e0> (a sun.nio.ch.KQueueSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:665) at org.apache.kafka.common.network.Selector.poll(Selector.java:396) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:460) at kafka.common.InterBrokerSendThread.doWork(InterBrokerSendThread.scala:74) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "transaction-log-manager-0" #50 daemon prio=5 os_prio=31 tid=0x00007f83e3d2a000 nid=0xb703 waiting on condition [0x00007000044e2000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00000007bcc57d40> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1093) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1067) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) at java.lang.Thread.run(Thread.java:745) Locked ownable synchronizers: - None "group-metadata-manager-0" #46 daemon prio=5 os_prio=31 tid=0x00007f83e294d800 nid=0xb503 waiting on condition [0x00007000043df000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00000007bca54c80> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1093) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1067) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) at java.lang.Thread.run(Thread.java:745) Locked ownable synchronizers: - None "ExpirationReaper-0-Rebalance" #45 prio=5 os_prio=31 tid=0x00007f83e6800000 nid=0xb303 waiting on condition [0x00007000042dc000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00000007bcc57fc0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.DelayQueue.poll(DelayQueue.java:259) at kafka.utils.timer.SystemTimer.advanceClock(Timer.scala:105) at kafka.server.DelayedOperationPurgatory.advanceClock(DelayedOperation.scala:393) at kafka.server.DelayedOperationPurgatory$ExpiredOperationReaper.doWork(DelayedOperation.scala:417) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ExpirationReaper-0-Heartbeat" #44 prio=5 os_prio=31 tid=0x00007f83e42a2800 nid=0xb103 waiting on condition [0x00007000041d9000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00000007bca54e60> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.DelayQueue.poll(DelayQueue.java:259) at kafka.utils.timer.SystemTimer.advanceClock(Timer.scala:105) at kafka.server.DelayedOperationPurgatory.advanceClock(DelayedOperation.scala:393) at kafka.server.DelayedOperationPurgatory$ExpiredOperationReaper.doWork(DelayedOperation.scala:417) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ExpirationReaper-0-topic" #43 prio=5 os_prio=31 tid=0x00007f83e42a1800 nid=0xaf03 waiting on condition [0x00007000040d6000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00000007bcc58230> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.DelayQueue.poll(DelayQueue.java:259) at kafka.utils.timer.SystemTimer.advanceClock(Timer.scala:105) at kafka.server.DelayedOperationPurgatory.advanceClock(DelayedOperation.scala:393) at kafka.server.DelayedOperationPurgatory$ExpiredOperationReaper.doWork(DelayedOperation.scala:417) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "controller-event-thread" #42 prio=5 os_prio=31 tid=0x00007f83e4293800 nid=0xad03 waiting on condition [0x0000700003fd3000] java.lang.Thread.State: WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00000007bcc584a0> (a java.util.concurrent.CountDownLatch$Sync) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer.parkAndCheckInterrupt(AbstractQueuedSynchronizer.java:836) at java.util.concurrent.locks.AbstractQueuedSynchronizer.doAcquireSharedInterruptibly(AbstractQueuedSynchronizer.java:997) at java.util.concurrent.locks.AbstractQueuedSynchronizer.acquireSharedInterruptibly(AbstractQueuedSynchronizer.java:1304) at java.util.concurrent.CountDownLatch.await(CountDownLatch.java:231) at kafka.zookeeper.ZooKeeperClient.handleRequests(ZooKeeperClient.scala:148) at kafka.zk.KafkaZkClient.retryRequestsUntilConnected(KafkaZkClient.scala:1439) at kafka.zk.KafkaZkClient.kafka$zk$KafkaZkClient$$retryRequestUntilConnected(KafkaZkClient.scala:1432) at kafka.zk.KafkaZkClient.registerZNodeChangeHandlerAndCheckExistence(KafkaZkClient.scala:1171) at kafka.controller.KafkaController$Reelect$.process(KafkaController.scala:1475) at kafka.controller.ControllerEventManager$ControllerEventThread$$anonfun$doWork$1.apply$mcV$sp(ControllerEventManager.scala:69) at kafka.controller.ControllerEventManager$ControllerEventThread$$anonfun$doWork$1.apply(ControllerEventManager.scala:69) at kafka.controller.ControllerEventManager$ControllerEventThread$$anonfun$doWork$1.apply(ControllerEventManager.scala:69) at kafka.metrics.KafkaTimer.time(KafkaTimer.scala:31) at kafka.controller.ControllerEventManager$ControllerEventThread.doWork(ControllerEventManager.scala:68) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "kafka-scheduler-9" #41 daemon prio=5 os_prio=31 tid=0x00007f83e3c97000 nid=0xab03 waiting on condition [0x0000700003ed0000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00000007bcb029e8> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1093) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1067) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) at java.lang.Thread.run(Thread.java:745) Locked ownable synchronizers: - None "LogDirFailureHandler" #40 prio=5 os_prio=31 tid=0x00007f83e3358800 nid=0xa903 waiting on condition [0x0000700003dcd000] java.lang.Thread.State: WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00000007bca552a0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) at java.util.concurrent.ArrayBlockingQueue.take(ArrayBlockingQueue.java:403) at kafka.server.LogDirFailureChannel.takeNextOfflineLogDir(LogDirFailureChannel.scala:57) at kafka.server.ReplicaManager$LogDirFailureHandler.doWork(ReplicaManager.scala:199) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "kafka-scheduler-8" #39 daemon prio=5 os_prio=31 tid=0x00007f83e3c98000 nid=0xa703 waiting on condition [0x0000700003cca000] java.lang.Thread.State: WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00000007bcb029e8> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1088) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1067) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) at java.lang.Thread.run(Thread.java:745) Locked ownable synchronizers: - None "kafka-scheduler-7" #38 daemon prio=5 os_prio=31 tid=0x00007f83e3358000 nid=0xa503 waiting on condition [0x0000700003bc7000] java.lang.Thread.State: WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00000007bcb029e8> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1088) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1067) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) at java.lang.Thread.run(Thread.java:745) Locked ownable synchronizers: - None "kafka-scheduler-6" #37 daemon prio=5 os_prio=31 tid=0x00007f83e3357000 nid=0xa303 waiting on condition [0x0000700003ac4000] java.lang.Thread.State: WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00000007bcb029e8> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1088) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1067) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) at java.lang.Thread.run(Thread.java:745) Locked ownable synchronizers: - None "kafka-scheduler-5" #36 daemon prio=5 os_prio=31 tid=0x00007f83e3356800 nid=0xa103 waiting on condition [0x00007000039c1000] java.lang.Thread.State: WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00000007bcb029e8> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1088) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1067) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) at java.lang.Thread.run(Thread.java:745) Locked ownable synchronizers: - None "ExpirationReaper-0-DeleteRecords" #35 prio=5 os_prio=31 tid=0x00007f83e334b800 nid=0x9f03 waiting on condition [0x00007000038be000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00000007bccac778> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.DelayQueue.poll(DelayQueue.java:259) at kafka.utils.timer.SystemTimer.advanceClock(Timer.scala:105) at kafka.server.DelayedOperationPurgatory.advanceClock(DelayedOperation.scala:393) at kafka.server.DelayedOperationPurgatory$ExpiredOperationReaper.doWork(DelayedOperation.scala:417) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ExpirationReaper-0-Fetch" #34 prio=5 os_prio=31 tid=0x00007f83e428f800 nid=0x9d03 waiting on condition [0x00007000037bb000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00000007bccac9e8> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.DelayQueue.poll(DelayQueue.java:259) at kafka.utils.timer.SystemTimer.advanceClock(Timer.scala:105) at kafka.server.DelayedOperationPurgatory.advanceClock(DelayedOperation.scala:393) at kafka.server.DelayedOperationPurgatory$ExpiredOperationReaper.doWork(DelayedOperation.scala:417) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ExpirationReaper-0-Produce" #33 prio=5 os_prio=31 tid=0x00007f83e428e800 nid=0x9b03 waiting on condition [0x00007000036b8000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00000007bcb02db8> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.DelayQueue.poll(DelayQueue.java:259) at kafka.utils.timer.SystemTimer.advanceClock(Timer.scala:105) at kafka.server.DelayedOperationPurgatory.advanceClock(DelayedOperation.scala:393) at kafka.server.DelayedOperationPurgatory$ExpiredOperationReaper.doWork(DelayedOperation.scala:417) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "kafka-network-thread-0-ListenerName(PLAINTEXT)-PLAINTEXT-2" #32 prio=5 os_prio=31 tid=0x00007f83e3353800 nid=0x9903 runnable [0x00007000035b5000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.KQueueArrayWrapper.kevent0(Native Method) at sun.nio.ch.KQueueArrayWrapper.poll(KQueueArrayWrapper.java:198) at sun.nio.ch.KQueueSelectorImpl.doSelect(KQueueSelectorImpl.java:117) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00000007bccaccb8> (a sun.nio.ch.Util$2) - locked <0x00000007bccaccc8> (a java.util.Collections$UnmodifiableSet) - locked <0x00000007bccacc68> (a sun.nio.ch.KQueueSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:665) at org.apache.kafka.common.network.Selector.poll(Selector.java:396) at kafka.network.Processor.poll(SocketServer.scala:618) at kafka.network.Processor.run(SocketServer.scala:535) at java.lang.Thread.run(Thread.java:745) Locked ownable synchronizers: - None "kafka-network-thread-0-ListenerName(PLAINTEXT)-PLAINTEXT-1" #31 prio=5 os_prio=31 tid=0x00007f83e3353000 nid=0x9703 runnable [0x00007000034b2000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.KQueueArrayWrapper.kevent0(Native Method) at sun.nio.ch.KQueueArrayWrapper.poll(KQueueArrayWrapper.java:198) at sun.nio.ch.KQueueSelectorImpl.doSelect(KQueueSelectorImpl.java:117) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00000007bcb03088> (a sun.nio.ch.Util$2) - locked <0x00000007bcb03098> (a java.util.Collections$UnmodifiableSet) - locked <0x00000007bcb03038> (a sun.nio.ch.KQueueSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:665) at org.apache.kafka.common.network.Selector.poll(Selector.java:396) at kafka.network.Processor.poll(SocketServer.scala:618) at kafka.network.Processor.run(SocketServer.scala:535) at java.lang.Thread.run(Thread.java:745) Locked ownable synchronizers: - None "kafka-network-thread-0-ListenerName(PLAINTEXT)-PLAINTEXT-0" #30 prio=5 os_prio=31 tid=0x00007f83e299c800 nid=0x9503 runnable [0x00007000033af000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.KQueueArrayWrapper.kevent0(Native Method) at sun.nio.ch.KQueueArrayWrapper.poll(KQueueArrayWrapper.java:198) at sun.nio.ch.KQueueSelectorImpl.doSelect(KQueueSelectorImpl.java:117) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00000007bcb03380> (a sun.nio.ch.Util$2) - locked <0x00000007bcb03390> (a java.util.Collections$UnmodifiableSet) - locked <0x00000007bcb03330> (a sun.nio.ch.KQueueSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:665) at org.apache.kafka.common.network.Selector.poll(Selector.java:396) at kafka.network.Processor.poll(SocketServer.scala:618) at kafka.network.Processor.run(SocketServer.scala:535) at java.lang.Thread.run(Thread.java:745) Locked ownable synchronizers: - None "kafka-socket-acceptor-ListenerName(PLAINTEXT)-PLAINTEXT-9092" #29 prio=5 os_prio=31 tid=0x00007f83e294d000 nid=0x9303 runnable [0x00007000032ac000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.KQueueArrayWrapper.kevent0(Native Method) at sun.nio.ch.KQueueArrayWrapper.poll(KQueueArrayWrapper.java:198) at sun.nio.ch.KQueueSelectorImpl.doSelect(KQueueSelectorImpl.java:117) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00000007bcb03678> (a sun.nio.ch.Util$2) - locked <0x00000007bcb03688> (a java.util.Collections$UnmodifiableSet) - locked <0x00000007bcb03628> (a sun.nio.ch.KQueueSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at kafka.network.Acceptor.run(SocketServer.scala:343) at java.lang.Thread.run(Thread.java:745) Locked ownable synchronizers: - None "kafka-log-cleaner-thread-0" #28 prio=5 os_prio=31 tid=0x00007f83e2930800 nid=0x9103 waiting on condition [0x00007000031a9000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00000007bcbc2a00> (a java.util.concurrent.CountDownLatch$Sync) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer.doAcquireSharedNanos(AbstractQueuedSynchronizer.java:1037) at java.util.concurrent.locks.AbstractQueuedSynchronizer.tryAcquireSharedNanos(AbstractQueuedSynchronizer.java:1328) at java.util.concurrent.CountDownLatch.await(CountDownLatch.java:277) at kafka.utils.ShutdownableThread.pause(ShutdownableThread.scala:69) at kafka.log.LogCleaner$CleanerThread.cleanOrSleep(LogCleaner.scala:330) at kafka.log.LogCleaner$CleanerThread.doWork(LogCleaner.scala:292) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "kafka-scheduler-4" #27 daemon prio=5 os_prio=31 tid=0x00007f83e581a800 nid=0x8f03 waiting on condition [0x00007000030a6000] java.lang.Thread.State: WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00000007bcb029e8> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1088) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1067) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) at java.lang.Thread.run(Thread.java:745) Locked ownable synchronizers: - None "kafka-scheduler-3" #26 daemon prio=5 os_prio=31 tid=0x00007f83e2930000 nid=0x8d03 waiting on condition [0x0000700002fa3000] java.lang.Thread.State: WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00000007bcb029e8> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1088) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1067) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) at java.lang.Thread.run(Thread.java:745) Locked ownable synchronizers: - None "kafka-scheduler-2" #25 daemon prio=5 os_prio=31 tid=0x00007f83e2910800 nid=0x8b03 waiting on condition [0x0000700002ea0000] java.lang.Thread.State: WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00000007bcb029e8> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1088) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1067) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) at java.lang.Thread.run(Thread.java:745) Locked ownable synchronizers: - None "kafka-scheduler-1" #24 daemon prio=5 os_prio=31 tid=0x00007f83e2916000 nid=0x8903 waiting on condition [0x0000700002d9d000] java.lang.Thread.State: WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00000007bcb029e8> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1088) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1067) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) at java.lang.Thread.run(Thread.java:745) Locked ownable synchronizers: - None "kafka-scheduler-0" #23 daemon prio=5 os_prio=31 tid=0x00007f83e2856000 nid=0x8703 waiting on condition [0x0000700002c9a000] java.lang.Thread.State: WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00000007bcb029e8> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1088) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1067) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) at java.lang.Thread.run(Thread.java:745) Locked ownable synchronizers: - None "ThrottledRequestReaper-Request" #22 prio=5 os_prio=31 tid=0x00007f83e42eb000 nid=0x8503 waiting on condition [0x0000700002b97000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00000007bccad860> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.DelayQueue.poll(DelayQueue.java:259) at kafka.server.ClientQuotaManager$ThrottledRequestReaper.doWork(ClientQuotaManager.scala:138) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ThrottledRequestReaper-Produce" #21 prio=5 os_prio=31 tid=0x00007f83e42ea800 nid=0x8303 waiting on condition [0x0000700002a94000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00000007bccada60> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.DelayQueue.poll(DelayQueue.java:259) at kafka.server.ClientQuotaManager$ThrottledRequestReaper.doWork(ClientQuotaManager.scala:138) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ThrottledRequestReaper-Fetch" #20 prio=5 os_prio=31 tid=0x00007f83e42ce800 nid=0x8103 waiting on condition [0x0000700002991000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00000007bccadc60> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.DelayQueue.poll(DelayQueue.java:259) at kafka.server.ClientQuotaManager$ThrottledRequestReaper.doWork(ClientQuotaManager.scala:138) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "SensorExpiryThread" #19 daemon prio=5 os_prio=31 tid=0x00007f83e3cf4800 nid=0x7f03 waiting on condition [0x000070000288e000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00000007bccade48> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1093) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1067) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) at java.lang.Thread.run(Thread.java:745) Locked ownable synchronizers: - None "main-EventThread" #18 daemon prio=5 os_prio=31 tid=0x00007f83e4285800 nid=0x7d03 waiting on condition [0x000070000278b000] java.lang.Thread.State: WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00000007bccadf30> (a java.util.concurrent.CountDownLatch$Sync) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer.parkAndCheckInterrupt(AbstractQueuedSynchronizer.java:836) at java.util.concurrent.locks.AbstractQueuedSynchronizer.doAcquireSharedInterruptibly(AbstractQueuedSynchronizer.java:997) at java.util.concurrent.locks.AbstractQueuedSynchronizer.acquireSharedInterruptibly(AbstractQueuedSynchronizer.java:1304) at java.util.concurrent.CountDownLatch.await(CountDownLatch.java:231) at kafka.controller.KafkaController$Expire.waitUntilProcessed(KafkaController.scala:1505) at kafka.controller.KafkaController$$anon$7.beforeInitializingSession(KafkaController.scala:163) at kafka.zookeeper.ZooKeeperClient$ZooKeeperClientWatcher$$anonfun$process$2$$anonfun$apply$mcV$sp$6.apply(ZooKeeperClient.scala:365) at kafka.zookeeper.ZooKeeperClient$ZooKeeperClientWatcher$$anonfun$process$2$$anonfun$apply$mcV$sp$6.apply(ZooKeeperClient.scala:365) at scala.collection.Iterator$class.foreach(Iterator.scala:891) at scala.collection.AbstractIterator.foreach(Iterator.scala:1334) at scala.collection.MapLike$DefaultValuesIterable.foreach(MapLike.scala:206) at kafka.zookeeper.ZooKeeperClient$ZooKeeperClientWatcher$$anonfun$process$2.apply$mcV$sp(ZooKeeperClient.scala:365) at kafka.zookeeper.ZooKeeperClient$ZooKeeperClientWatcher$$anonfun$process$2.apply(ZooKeeperClient.scala:363) at kafka.zookeeper.ZooKeeperClient$ZooKeeperClientWatcher$$anonfun$process$2.apply(ZooKeeperClient.scala:363) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:250) at kafka.utils.CoreUtils$.inWriteLock(CoreUtils.scala:258) at kafka.zookeeper.ZooKeeperClient$ZooKeeperClientWatcher$.process(ZooKeeperClient.scala:363) at org.apache.zookeeper.ClientCnxn$EventThread.processEvent(ClientCnxn.java:531) at org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:506) Locked ownable synchronizers: - <0x0000000780054860> (a java.util.concurrent.locks.ReentrantReadWriteLock$NonfairSync) "metrics-meter-tick-thread-2" #16 daemon prio=5 os_prio=31 tid=0x00007f83e3ce7800 nid=0x7903 waiting on condition [0x0000700002585000] java.lang.Thread.State: WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00000007886906f8> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1088) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1067) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) at java.lang.Thread.run(Thread.java:745) Locked ownable synchronizers: - None "metrics-meter-tick-thread-1" #15 daemon prio=5 os_prio=31 tid=0x00007f83e3ce3000 nid=0x7703 waiting on condition [0x0000700002482000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00000007886906f8> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1093) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1067) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) at java.lang.Thread.run(Thread.java:745) Locked ownable synchronizers: - None "RMI TCP Accept-0" #12 daemon prio=5 os_prio=31 tid=0x00007f83e30f1000 nid=0x730f runnable [0x000070000227c000] java.lang.Thread.State: RUNNABLE at java.net.PlainSocketImpl.socketAccept(Native Method) at java.net.AbstractPlainSocketImpl.accept(AbstractPlainSocketImpl.java:409) at java.net.ServerSocket.implAccept(ServerSocket.java:545) at java.net.ServerSocket.accept(ServerSocket.java:513) at sun.management.jmxremote.LocalRMIServerSocketFactory$1.accept(LocalRMIServerSocketFactory.java:52) at sun.rmi.transport.tcp.TCPTransport$AcceptLoop.executeAcceptLoop(TCPTransport.java:400) at sun.rmi.transport.tcp.TCPTransport$AcceptLoop.run(TCPTransport.java:372) at java.lang.Thread.run(Thread.java:745) Locked ownable synchronizers: - None "Service Thread" #10 daemon prio=9 os_prio=31 tid=0x00007f83e30ca000 nid=0x6d03 runnable [0x0000000000000000] java.lang.Thread.State: RUNNABLE Locked ownable synchronizers: - None "C1 CompilerThread3" #9 daemon prio=9 os_prio=31 tid=0x00007f83e2819000 nid=0x6b03 waiting on condition [0x0000000000000000] java.lang.Thread.State: RUNNABLE Locked ownable synchronizers: - None "C2 CompilerThread2" #8 daemon prio=9 os_prio=31 tid=0x00007f83e3a21000 nid=0x6903 waiting on condition [0x0000000000000000] java.lang.Thread.State: RUNNABLE Locked ownable synchronizers: - None "C2 CompilerThread1" #7 daemon prio=9 os_prio=31 tid=0x00007f83e30be000 nid=0x6703 waiting on condition [0x0000000000000000] java.lang.Thread.State: RUNNABLE Locked ownable synchronizers: - None "C2 CompilerThread0" #6 daemon prio=9 os_prio=31 tid=0x00007f83e4006800 nid=0x6503 waiting on condition [0x0000000000000000] java.lang.Thread.State: RUNNABLE Locked ownable synchronizers: - None "Signal Dispatcher" #5 daemon prio=9 os_prio=31 tid=0x00007f83e30bd800 nid=0x6303 runnable [0x0000000000000000] java.lang.Thread.State: RUNNABLE Locked ownable synchronizers: - None "Surrogate Locker Thread (Concurrent GC)" #4 daemon prio=9 os_prio=31 tid=0x00007f83e30c9800 nid=0x570b waiting on condition [0x0000000000000000] java.lang.Thread.State: RUNNABLE Locked ownable synchronizers: - None "Finalizer" #3 daemon prio=8 os_prio=31 tid=0x00007f83e3080800 nid=0x5003 in Object.wait() [0x000070000195e000] java.lang.Thread.State: WAITING (on object monitor) at java.lang.Object.wait(Native Method) - waiting on <0x0000000788707c30> (a java.lang.ref.ReferenceQueue$Lock) at java.lang.ref.ReferenceQueue.remove(ReferenceQueue.java:143) - locked <0x0000000788707c30> (a java.lang.ref.ReferenceQueue$Lock) at java.lang.ref.ReferenceQueue.remove(ReferenceQueue.java:164) at java.lang.ref.Finalizer$FinalizerThread.run(Finalizer.java:209) Locked ownable synchronizers: - None "Reference Handler" #2 daemon prio=10 os_prio=31 tid=0x00007f83e307e000 nid=0x4e03 in Object.wait() [0x000070000185b000] java.lang.Thread.State: WAITING (on object monitor) at java.lang.Object.wait(Native Method) - waiting on <0x0000000788707c20> (a java.lang.ref.Reference$Lock) at java.lang.Object.wait(Object.java:502) at java.lang.ref.Reference.tryHandlePending(Reference.java:191) - locked <0x0000000788707c20> (a java.lang.ref.Reference$Lock) at java.lang.ref.Reference$ReferenceHandler.run(Reference.java:153) Locked ownable synchronizers: - None "main" #1 prio=5 os_prio=31 tid=0x00007f83e3802800 nid=0x1703 waiting on condition [0x0000700000219000] java.lang.Thread.State: WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00000007bca55548> (a java.util.concurrent.CountDownLatch$Sync) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer.parkAndCheckInterrupt(AbstractQueuedSynchronizer.java:836) at java.util.concurrent.locks.AbstractQueuedSynchronizer.doAcquireSharedInterruptibly(AbstractQueuedSynchronizer.java:997) at java.util.concurrent.locks.AbstractQueuedSynchronizer.acquireSharedInterruptibly(AbstractQueuedSynchronizer.java:1304) at java.util.concurrent.CountDownLatch.await(CountDownLatch.java:231) at kafka.server.KafkaServer.awaitShutdown(KafkaServer.scala:628) at kafka.server.KafkaServerStartable.awaitShutdown(KafkaServerStartable.scala:65) at kafka.Kafka$.main(Kafka.scala:93) at kafka.Kafka.main(Kafka.scala) Locked ownable synchronizers: - None "VM Thread" os_prio=31 tid=0x00007f83e3079800 nid=0x4c03 runnable "Gang worker#0 (Parallel GC Threads)" os_prio=31 tid=0x00007f83e3811000 nid=0x2403 runnable "Gang worker#1 (Parallel GC Threads)" os_prio=31 tid=0x00007f83e3811800 nid=0x2603 runnable "Gang worker#2 (Parallel GC Threads)" os_prio=31 tid=0x00007f83e4001800 nid=0x2803 runnable "Gang worker#3 (Parallel GC Threads)" os_prio=31 tid=0x00007f83e3812000 nid=0x2a03 runnable "Gang worker#4 (Parallel GC Threads)" os_prio=31 tid=0x00007f83e3002000 nid=0x2c03 runnable "Gang worker#5 (Parallel GC Threads)" os_prio=31 tid=0x00007f83e3003000 nid=0x2e03 runnable "Gang worker#6 (Parallel GC Threads)" os_prio=31 tid=0x00007f83e3003800 nid=0x3003 runnable "Gang worker#7 (Parallel GC Threads)" os_prio=31 tid=0x00007f83e3813000 nid=0x3203 runnable "G1 Main Concurrent Mark GC Thread" os_prio=31 tid=0x00007f83e2816800 nid=0x4603 runnable "Gang worker#0 (G1 Parallel Marking Threads)" os_prio=31 tid=0x00007f83e2817800 nid=0x4803 runnable "Gang worker#1 (G1 Parallel Marking Threads)" os_prio=31 tid=0x00007f83e3005800 nid=0x4a03 runnable "G1 Concurrent Refinement Thread#0" os_prio=31 tid=0x00007f83e3815800 nid=0x4403 runnable "G1 Concurrent Refinement Thread#1" os_prio=31 tid=0x00007f83e4008800 nid=0x4203 runnable "G1 Concurrent Refinement Thread#2" os_prio=31 tid=0x00007f83e3815000 nid=0x4003 runnable "G1 Concurrent Refinement Thread#3" os_prio=31 tid=0x00007f83e3814000 nid=0x3e03 runnable "G1 Concurrent Refinement Thread#4" os_prio=31 tid=0x00007f83e3813800 nid=0x3c03 runnable "G1 Concurrent Refinement Thread#5" os_prio=31 tid=0x00007f83e4003000 nid=0x3a03 runnable "G1 Concurrent Refinement Thread#6" os_prio=31 tid=0x00007f83e4002000 nid=0x3803 runnable "G1 Concurrent Refinement Thread#7" os_prio=31 tid=0x00007f83e3005000 nid=0x3603 runnable "G1 Concurrent Refinement Thread#8" os_prio=31 tid=0x00007f83e3004000 nid=0x3403 runnable "VM Periodic Task Thread" os_prio=31 tid=0x00007f83e30f6800 nid=0x7503 waiting on condition JNI global references: 266