2016-08-13 03:01:20 Full thread dump Java HotSpot(TM) 64-Bit Server VM (25.92-b14 mixed mode): "Attach Listener" #114 daemon prio=9 os_prio=0 tid=0x00007f125c001000 nid=0xee4 waiting on condition [0x0000000000000000] java.lang.Thread.State: RUNNABLE "Thread-2" #29 prio=5 os_prio=0 tid=0x00007f0a70008000 nid=0xfecbf in Object.wait() [0x00007f0b166e5000] java.lang.Thread.State: WAITING (on object monitor) at java.lang.Object.wait(Native Method) at java.lang.Thread.join(Thread.java:1245) - locked <0x00000003e5c46960> (a java.lang.Thread) at java.lang.Thread.join(Thread.java:1319) at kafka.server.KafkaRequestHandlerPool$$anonfun$shutdown$3.apply(KafkaRequestHandler.scala:92) at kafka.server.KafkaRequestHandlerPool$$anonfun$shutdown$3.apply(KafkaRequestHandler.scala:91) at scala.collection.IndexedSeqOptimized$class.foreach(IndexedSeqOptimized.scala:33) at scala.collection.mutable.ArrayOps$ofRef.foreach(ArrayOps.scala:186) at kafka.server.KafkaRequestHandlerPool.shutdown(KafkaRequestHandler.scala:91) at kafka.server.KafkaServer$$anonfun$shutdown$3.apply$mcV$sp(KafkaServer.scala:559) at kafka.utils.CoreUtils$.swallow(CoreUtils.scala:79) at kafka.utils.Logging$class.swallowWarn(Logging.scala:92) at kafka.utils.CoreUtils$.swallowWarn(CoreUtils.scala:51) at kafka.utils.Logging$class.swallow(Logging.scala:94) at kafka.utils.CoreUtils$.swallow(CoreUtils.scala:51) at kafka.server.KafkaServer.shutdown(KafkaServer.scala:559) at kafka.server.KafkaServerStartable.shutdown(KafkaServerStartable.scala:49) at kafka.Kafka$$anon$1.run(Kafka.scala:63) "ReplicaFetcherThread-2-5" #113 prio=5 os_prio=0 tid=0x00007f0acc11a800 nid=0xfd550 runnable [0x00007f0af4dfe000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00000003e65ebe30> (a sun.nio.ch.Util$2) - locked <0x00000003e65ebe20> (a java.util.Collections$UnmodifiableSet) - locked <0x00000003e65eb988> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:425) at org.apache.kafka.common.network.Selector.poll(Selector.java:254) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:256) at kafka.utils.NetworkClientBlockingOps$.recurse$1(NetworkClientBlockingOps.scala:128) at kafka.utils.NetworkClientBlockingOps$.kafka$utils$NetworkClientBlockingOps$$pollUntilFound$extension(NetworkClientBlockingOps.scala:139) at kafka.utils.NetworkClientBlockingOps$.blockingSendAndReceive$extension(NetworkClientBlockingOps.scala:80) at kafka.server.ReplicaFetcherThread.sendRequest(ReplicaFetcherThread.scala:242) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:227) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:42) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:106) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:97) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:63) "ReplicaFetcherThread-3-2" #112 prio=5 os_prio=0 tid=0x00007f0acc100000 nid=0xfd54f in Object.wait() [0x00007f0b141d7000] java.lang.Thread.State: WAITING (on object monitor) at java.lang.Object.wait(Native Method) at java.lang.Thread.join(Thread.java:1245) - locked <0x00000003d8269bc8> (a kafka.Kafka$$anon$1) at java.lang.Thread.join(Thread.java:1319) at java.lang.ApplicationShutdownHooks.runHooks(ApplicationShutdownHooks.java:106) at java.lang.ApplicationShutdownHooks$1.run(ApplicationShutdownHooks.java:46) at java.lang.Shutdown.runHooks(Shutdown.java:123) at java.lang.Shutdown.sequence(Shutdown.java:167) at java.lang.Shutdown.exit(Shutdown.java:212) - locked <0x00000003d8106e88> (a java.lang.Class for java.lang.Shutdown) at java.lang.Runtime.exit(Runtime.java:109) at java.lang.System.exit(System.java:971) at kafka.server.ReplicaFetcherThread.handleOffsetOutOfRange(ReplicaFetcherThread.scala:179) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:156) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:126) at scala.Option.foreach(Option.scala:257) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:126) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:124) at scala.collection.mutable.HashMap$$anonfun$foreach$1.apply(HashMap.scala:99) at scala.collection.mutable.HashMap$$anonfun$foreach$1.apply(HashMap.scala:99) at scala.collection.mutable.HashTable$class.foreachEntry(HashTable.scala:230) at scala.collection.mutable.HashMap.foreachEntry(HashMap.scala:40) at scala.collection.mutable.HashMap.foreach(HashMap.scala:99) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply$mcV$sp(AbstractFetcherThread.scala:124) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:124) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:124) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:262) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:122) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:97) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:63) "ReplicaFetcherThread-1-2" #111 prio=5 os_prio=0 tid=0x00007f0acc0e4800 nid=0xfd54e runnable [0x00007f0b142d9000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00000003e64fd118> (a sun.nio.ch.Util$2) - locked <0x00000003e64fd108> (a java.util.Collections$UnmodifiableSet) - locked <0x00000003e64fd128> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:425) at org.apache.kafka.common.network.Selector.poll(Selector.java:254) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:256) at kafka.utils.NetworkClientBlockingOps$.recurse$1(NetworkClientBlockingOps.scala:128) at kafka.utils.NetworkClientBlockingOps$.kafka$utils$NetworkClientBlockingOps$$pollUntilFound$extension(NetworkClientBlockingOps.scala:139) at kafka.utils.NetworkClientBlockingOps$.blockingSendAndReceive$extension(NetworkClientBlockingOps.scala:80) at kafka.server.ReplicaFetcherThread.sendRequest(ReplicaFetcherThread.scala:242) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:227) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:42) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:106) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:97) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:63) "ReplicaFetcherThread-2-2" #110 prio=5 os_prio=0 tid=0x00007f0acc0ca000 nid=0xfd54d runnable [0x00007f0b143da000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00000003e64fb120> (a sun.nio.ch.Util$2) - locked <0x00000003e64fb110> (a java.util.Collections$UnmodifiableSet) - locked <0x00000003e64fb130> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:425) at org.apache.kafka.common.network.Selector.poll(Selector.java:254) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:256) at kafka.utils.NetworkClientBlockingOps$.recurse$1(NetworkClientBlockingOps.scala:128) at kafka.utils.NetworkClientBlockingOps$.kafka$utils$NetworkClientBlockingOps$$pollUntilFound$extension(NetworkClientBlockingOps.scala:139) at kafka.utils.NetworkClientBlockingOps$.blockingSendAndReceive$extension(NetworkClientBlockingOps.scala:80) at kafka.server.ReplicaFetcherThread.sendRequest(ReplicaFetcherThread.scala:242) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:227) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:42) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:106) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:97) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:63) "ReplicaFetcherThread-0-5" #109 prio=5 os_prio=0 tid=0x00007f0acc0af000 nid=0xfd54c runnable [0x00007f0b144db000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00000003e64fb270> (a sun.nio.ch.Util$2) - locked <0x00000003e64fb260> (a java.util.Collections$UnmodifiableSet) - locked <0x00000003e64fb280> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:425) at org.apache.kafka.common.network.Selector.poll(Selector.java:254) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:256) at kafka.utils.NetworkClientBlockingOps$.recurse$1(NetworkClientBlockingOps.scala:128) at kafka.utils.NetworkClientBlockingOps$.kafka$utils$NetworkClientBlockingOps$$pollUntilFound$extension(NetworkClientBlockingOps.scala:139) at kafka.utils.NetworkClientBlockingOps$.blockingSendAndReceive$extension(NetworkClientBlockingOps.scala:80) at kafka.server.ReplicaFetcherThread.sendRequest(ReplicaFetcherThread.scala:242) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:227) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:42) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:106) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:97) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:63) "ReplicaFetcherThread-0-2" #108 prio=5 os_prio=0 tid=0x00007f0acc094800 nid=0xfd54b runnable [0x00007f0b145dc000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00000003e6502848> (a sun.nio.ch.Util$2) - locked <0x00000003e6502838> (a java.util.Collections$UnmodifiableSet) - locked <0x00000003e6502858> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:425) at org.apache.kafka.common.network.Selector.poll(Selector.java:254) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:256) at kafka.utils.NetworkClientBlockingOps$.recurse$1(NetworkClientBlockingOps.scala:128) at kafka.utils.NetworkClientBlockingOps$.kafka$utils$NetworkClientBlockingOps$$pollUntilFound$extension(NetworkClientBlockingOps.scala:139) at kafka.utils.NetworkClientBlockingOps$.blockingSendAndReceive$extension(NetworkClientBlockingOps.scala:80) at kafka.server.ReplicaFetcherThread.sendRequest(ReplicaFetcherThread.scala:242) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:227) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:42) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:106) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:97) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:63) "ReplicaFetcherThread-3-5" #107 prio=5 os_prio=0 tid=0x00007f0acc07b000 nid=0xfd54a runnable [0x00007f0b146dd000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00000003e6508528> (a sun.nio.ch.Util$2) - locked <0x00000003e6508518> (a java.util.Collections$UnmodifiableSet) - locked <0x00000003e6508538> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:425) at org.apache.kafka.common.network.Selector.poll(Selector.java:254) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:256) at kafka.utils.NetworkClientBlockingOps$.recurse$1(NetworkClientBlockingOps.scala:128) at kafka.utils.NetworkClientBlockingOps$.kafka$utils$NetworkClientBlockingOps$$pollUntilFound$extension(NetworkClientBlockingOps.scala:139) at kafka.utils.NetworkClientBlockingOps$.blockingSendAndReceive$extension(NetworkClientBlockingOps.scala:80) at kafka.server.ReplicaFetcherThread.sendRequest(ReplicaFetcherThread.scala:242) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:227) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:42) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:106) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:97) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:63) "ReplicaFetcherThread-1-5" #106 prio=5 os_prio=0 tid=0x00007f0acc05d000 nid=0xfd549 runnable [0x00007f0b147de000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00000003e64fd268> (a sun.nio.ch.Util$2) - locked <0x00000003e64fd258> (a java.util.Collections$UnmodifiableSet) - locked <0x00000003e64fd278> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:425) at org.apache.kafka.common.network.Selector.poll(Selector.java:254) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:256) at kafka.utils.NetworkClientBlockingOps$.recurse$1(NetworkClientBlockingOps.scala:128) at kafka.utils.NetworkClientBlockingOps$.kafka$utils$NetworkClientBlockingOps$$pollUntilFound$extension(NetworkClientBlockingOps.scala:139) at kafka.utils.NetworkClientBlockingOps$.blockingSendAndReceive$extension(NetworkClientBlockingOps.scala:80) at kafka.server.ReplicaFetcherThread.sendRequest(ReplicaFetcherThread.scala:242) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:227) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:42) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:106) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:97) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:63) "kafka-scheduler-9" #105 daemon prio=5 os_prio=0 tid=0x00007f11e0002800 nid=0xfd51c waiting on condition [0x00007f0b15cdf000] java.lang.Thread.State: WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00000003d81374b0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1088) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1067) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) at java.lang.Thread.run(Thread.java:745) "kafka-scheduler-8" #104 daemon prio=5 os_prio=0 tid=0x00007f11e401b000 nid=0xfd511 waiting on condition [0x00007f0b15de0000] java.lang.Thread.State: WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00000003d81374b0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1088) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1067) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) at java.lang.Thread.run(Thread.java:745) "kafka-scheduler-7" #103 daemon prio=5 os_prio=0 tid=0x00007f0b54001000 nid=0xfd505 waiting on condition [0x00007f0b160e1000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00000003d81374b0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1093) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1067) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) at java.lang.Thread.run(Thread.java:745) "JMX server connection timeout 102" #102 daemon prio=5 os_prio=0 tid=0x00007f0aa400f800 nid=0xfd4fc in Object.wait() [0x00007f0b161e2000] java.lang.Thread.State: TIMED_WAITING (on object monitor) at java.lang.Object.wait(Native Method) at com.sun.jmx.remote.internal.ServerCommunicatorAdmin$Timeout.run(ServerCommunicatorAdmin.java:168) - locked <0x00000003e7093c40> (a [I) at java.lang.Thread.run(Thread.java:745) "RMI TCP Connection(19)-5.45.227.234" #101 daemon prio=5 os_prio=0 tid=0x00007f1208005800 nid=0xfd4fb runnable [0x00007f0b162e2000] java.lang.Thread.State: RUNNABLE at java.net.SocketInputStream.socketRead0(Native Method) at java.net.SocketInputStream.socketRead(SocketInputStream.java:116) at java.net.SocketInputStream.read(SocketInputStream.java:170) at java.net.SocketInputStream.read(SocketInputStream.java:141) at java.io.BufferedInputStream.fill(BufferedInputStream.java:246) at java.io.BufferedInputStream.read(BufferedInputStream.java:265) - locked <0x00000004e083e2d0> (a java.io.BufferedInputStream) at java.io.FilterInputStream.read(FilterInputStream.java:83) at sun.rmi.transport.tcp.TCPTransport.handleMessages(TCPTransport.java:550) at sun.rmi.transport.tcp.TCPTransport$ConnectionHandler.run0(TCPTransport.java:826) at sun.rmi.transport.tcp.TCPTransport$ConnectionHandler.lambda$run$0(TCPTransport.java:683) at sun.rmi.transport.tcp.TCPTransport$ConnectionHandler$$Lambda$2/788663585.run(Unknown Source) at java.security.AccessController.doPrivileged(Native Method) at sun.rmi.transport.tcp.TCPTransport$ConnectionHandler.run(TCPTransport.java:682) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) at java.lang.Thread.run(Thread.java:745) "RMI Scheduler(0)" #100 daemon prio=5 os_prio=0 tid=0x00007f0aac010000 nid=0xfd4f9 waiting on condition [0x00007f0b163e4000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00000003d8240ad0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1093) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1067) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) at java.lang.Thread.run(Thread.java:745) "kafka-request-handler-0" #67 daemon prio=5 os_prio=0 tid=0x00007f138ce35000 nid=0xfd4d7 waiting on condition [0x00007f0b8c758000] java.lang.Thread.State: WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00000003e65c91c0> (a java.util.concurrent.locks.ReentrantLock$NonfairSync) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer.parkAndCheckInterrupt(AbstractQueuedSynchronizer.java:836) at java.util.concurrent.locks.AbstractQueuedSynchronizer.doAcquireInterruptibly(AbstractQueuedSynchronizer.java:897) at java.util.concurrent.locks.AbstractQueuedSynchronizer.acquireInterruptibly(AbstractQueuedSynchronizer.java:1222) at java.util.concurrent.locks.ReentrantLock.lockInterruptibly(ReentrantLock.java:335) at kafka.server.AbstractFetcherThread.removePartitions(AbstractFetcherThread.scala:212) at kafka.server.AbstractFetcherManager$$anonfun$removeFetcherForPartitions$2.apply(AbstractFetcherManager.scala:101) at kafka.server.AbstractFetcherManager$$anonfun$removeFetcherForPartitions$2.apply(AbstractFetcherManager.scala:100) at scala.collection.TraversableLike$WithFilter$$anonfun$foreach$1.apply(TraversableLike.scala:778) at scala.collection.mutable.HashMap$$anonfun$foreach$1.apply(HashMap.scala:99) at scala.collection.mutable.HashMap$$anonfun$foreach$1.apply(HashMap.scala:99) at scala.collection.mutable.HashTable$class.foreachEntry(HashTable.scala:230) at scala.collection.mutable.HashMap.foreachEntry(HashMap.scala:40) at scala.collection.mutable.HashMap.foreach(HashMap.scala:99) at scala.collection.TraversableLike$WithFilter.foreach(TraversableLike.scala:777) at kafka.server.AbstractFetcherManager.removeFetcherForPartitions(AbstractFetcherManager.scala:100) - locked <0x00000003e5f10bb8> (a java.lang.Object) at kafka.server.ReplicaManager.stopReplicas(ReplicaManager.scala:259) - locked <0x00000003e5c51d58> (a java.lang.Object) at kafka.server.KafkaApis.handleStopReplicaRequest(KafkaApis.scala:153) at kafka.server.KafkaApis.handle(KafkaApis.scala:77) at kafka.server.KafkaRequestHandler.run(KafkaRequestHandler.scala:60) at java.lang.Thread.run(Thread.java:745) "ThrottledRequestReaper-Fetch" #66 prio=5 os_prio=0 tid=0x00007f138ce31000 nid=0xfd4d6 waiting on condition [0x00007f0b8c859000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00000003e5c50568> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.DelayQueue.poll(DelayQueue.java:259) at kafka.server.ClientQuotaManager$ThrottledRequestReaper.doWork(ClientQuotaManager.scala:91) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:63) "ThrottledRequestReaper-Produce" #65 prio=5 os_prio=0 tid=0x00007f138ce2f800 nid=0xfd4d5 waiting on condition [0x00007f0b8c95a000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00000003e5c46b10> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.DelayQueue.poll(DelayQueue.java:259) at kafka.server.ClientQuotaManager$ThrottledRequestReaper.doWork(ClientQuotaManager.scala:91) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:63) "ExpirationReaper-4" #64 prio=5 os_prio=0 tid=0x00007f138ce11800 nid=0xfd4d4 waiting on condition [0x00007f0b8ca5b000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00000003e5c4db38> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.DelayQueue.poll(DelayQueue.java:259) at kafka.utils.timer.Timer.advanceClock(Timer.scala:67) at kafka.server.DelayedOperationPurgatory$ExpiredOperationReaper.doWork(DelayedOperation.scala:349) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:63) "ExpirationReaper-4" #63 prio=5 os_prio=0 tid=0x00007f138ce10000 nid=0xfd4d3 waiting on condition [0x00007f0b8cb5c000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00000003e5c4dfa0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.DelayQueue.poll(DelayQueue.java:259) at kafka.utils.timer.Timer.advanceClock(Timer.scala:67) at kafka.server.DelayedOperationPurgatory$ExpiredOperationReaper.doWork(DelayedOperation.scala:349) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:63) "group-metadata-manager-0" #62 daemon prio=5 os_prio=0 tid=0x00007f138ce0e800 nid=0xfd4d2 waiting on condition [0x00007f0b8cc5d000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00000003e5c4b340> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1093) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1067) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) at java.lang.Thread.run(Thread.java:745) "kafka-scheduler-6" #61 daemon prio=5 os_prio=0 tid=0x00007f11f4060000 nid=0xfd4d0 waiting on condition [0x00007f0b8cf5e000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00000003d81374b0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1093) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1067) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) at java.lang.Thread.run(Thread.java:745) "kafka-scheduler-5" #60 daemon prio=5 os_prio=0 tid=0x00007f11e4017000 nid=0xfd4cf waiting on condition [0x00007f0b8d05f000] java.lang.Thread.State: WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00000003d81374b0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1088) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1067) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) at java.lang.Thread.run(Thread.java:745) "kafka-scheduler-4" #59 daemon prio=5 os_prio=0 tid=0x00007f138cdbc000 nid=0xfd4ce waiting on condition [0x00007f0b8d160000] java.lang.Thread.State: WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00000003d81374b0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1088) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1067) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) at java.lang.Thread.run(Thread.java:745) "kafka-scheduler-3" #58 daemon prio=5 os_prio=0 tid=0x00007f138cdbb000 nid=0xfd4cd waiting on condition [0x00007f0b8d261000] java.lang.Thread.State: WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00000003d81374b0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1088) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1067) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) at java.lang.Thread.run(Thread.java:745) "metrics-meter-tick-thread-2" #57 daemon prio=5 os_prio=0 tid=0x00007f138cd9e800 nid=0xfd4cb waiting on condition [0x00007f0b8d362000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00000003e5c4afc8> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1093) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1067) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) at java.lang.Thread.run(Thread.java:745) "ExpirationReaper-4" #56 prio=5 os_prio=0 tid=0x00007f138cd9e000 nid=0xfd4ca waiting on condition [0x00007f0b8d463000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00000003e5c457c0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.DelayQueue.poll(DelayQueue.java:259) at kafka.utils.timer.Timer.advanceClock(Timer.scala:67) at kafka.server.DelayedOperationPurgatory$ExpiredOperationReaper.doWork(DelayedOperation.scala:349) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:63) "ExpirationReaper-4" #55 prio=5 os_prio=0 tid=0x00007f138cd9d000 nid=0xfd4c9 waiting on condition [0x00007f0b8d564000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00000003e5c4b770> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.DelayQueue.poll(DelayQueue.java:259) at kafka.utils.timer.Timer.advanceClock(Timer.scala:67) at kafka.server.DelayedOperationPurgatory$ExpiredOperationReaper.doWork(DelayedOperation.scala:349) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:63) "kafka-log-cleaner-thread-0" #42 prio=5 os_prio=0 tid=0x00007f138c888000 nid=0xfd4bf waiting on condition [0x00007f0b8df6e000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00000003e5c4cc00> (a java.util.concurrent.CountDownLatch$Sync) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer.doAcquireSharedNanos(AbstractQueuedSynchronizer.java:1037) at java.util.concurrent.locks.AbstractQueuedSynchronizer.tryAcquireSharedNanos(AbstractQueuedSynchronizer.java:1328) at java.util.concurrent.CountDownLatch.await(CountDownLatch.java:277) at kafka.log.LogCleaner$CleanerThread.cleanOrSleep(LogCleaner.scala:225) at kafka.log.LogCleaner$CleanerThread.doWork(LogCleaner.scala:208) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:63) "kafka-scheduler-2" #45 daemon prio=5 os_prio=0 tid=0x00007f138c4e7000 nid=0xfd4be waiting on condition [0x00007f120defd000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00000003d81374b0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1093) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1067) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) at java.lang.Thread.run(Thread.java:745) "kafka-scheduler-1" #44 daemon prio=5 os_prio=0 tid=0x00007f138c4e6000 nid=0xfd4bd waiting on condition [0x00007f120d7fa000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00000003d81374b0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1093) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1067) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) at java.lang.Thread.run(Thread.java:745) "kafka-scheduler-0" #43 daemon prio=5 os_prio=0 tid=0x00007f138c4e5000 nid=0xfd4bc waiting on condition [0x00007f120c2f6000] java.lang.Thread.State: WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00000003d81374b0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1088) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1067) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) at java.lang.Thread.run(Thread.java:745) "metrics-meter-tick-thread-1" #41 daemon prio=5 os_prio=0 tid=0x00007f138cb76800 nid=0xfd4ba waiting on condition [0x00007f120c1ce000] java.lang.Thread.State: WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00000003e5c4afc8> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1088) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1067) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) at java.lang.Thread.run(Thread.java:745) "main-EventThread" #36 daemon prio=5 os_prio=0 tid=0x00007f138cadd000 nid=0xfd450 waiting on condition [0x00007f120dcfb000] java.lang.Thread.State: WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00000003d8006790> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) at org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:494) "main-SendThread(zk03ep.stat.yandex.net:2181)" #35 daemon prio=5 os_prio=0 tid=0x00007f138cabd000 nid=0xfd44f runnable [0x00007f120ddfc000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00000003d8000ec0> (a sun.nio.ch.Util$2) - locked <0x00000003d8000eb0> (a java.util.Collections$UnmodifiableSet) - locked <0x00000003d8000e68> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.zookeeper.ClientCnxnSocketNIO.doTransport(ClientCnxnSocketNIO.java:349) at org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1081) "ZkClient-EventThread-34-zk01ep.stat.yandex.net:2181,zk02ep.stat.yandex.net:2181,zk03ep.stat.yandex.net:2181/kafka-iva-et" #34 daemon prio=5 os_prio=0 tid=0x00007f138ca9c800 nid=0xfd44e waiting on condition [0x00007f120dffe000] java.lang.Thread.State: WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00000003d8024420> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) at org.I0Itec.zkclient.ZkEventThread.run(ZkEventThread.java:67) "SensorExpiryThread" #30 daemon prio=5 os_prio=0 tid=0x00007f138ca90800 nid=0xfd449 waiting on condition [0x00007f126c169000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00000003d8006990> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1093) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1067) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) at java.lang.Thread.run(Thread.java:745) "RMI TCP Accept-0" #27 daemon prio=5 os_prio=0 tid=0x00007f138c7c2800 nid=0xfd444 runnable [0x00007f126c56b000] java.lang.Thread.State: RUNNABLE at java.net.PlainSocketImpl.socketAccept(Native Method) at java.net.AbstractPlainSocketImpl.accept(AbstractPlainSocketImpl.java:409) at java.net.ServerSocket.implAccept(ServerSocket.java:545) at java.net.ServerSocket.accept(ServerSocket.java:513) at sun.management.jmxremote.LocalRMIServerSocketFactory$1.accept(LocalRMIServerSocketFactory.java:52) at sun.rmi.transport.tcp.TCPTransport$AcceptLoop.executeAcceptLoop(TCPTransport.java:400) at sun.rmi.transport.tcp.TCPTransport$AcceptLoop.run(TCPTransport.java:372) at java.lang.Thread.run(Thread.java:745) "RMI TCP Accept-9999" #26 daemon prio=5 os_prio=0 tid=0x00007f138c7b9800 nid=0xfd443 runnable [0x00007f126c66c000] java.lang.Thread.State: RUNNABLE at java.net.PlainSocketImpl.socketAccept(Native Method) at java.net.AbstractPlainSocketImpl.accept(AbstractPlainSocketImpl.java:409) at java.net.ServerSocket.implAccept(ServerSocket.java:545) at java.net.ServerSocket.accept(ServerSocket.java:513) at sun.rmi.transport.tcp.TCPTransport$AcceptLoop.executeAcceptLoop(TCPTransport.java:400) at sun.rmi.transport.tcp.TCPTransport$AcceptLoop.run(TCPTransport.java:372) at java.lang.Thread.run(Thread.java:745) "RMI TCP Accept-0" #25 daemon prio=5 os_prio=0 tid=0x00007f138c7a3000 nid=0xfd442 runnable [0x00007f126c76d000] java.lang.Thread.State: RUNNABLE at java.net.PlainSocketImpl.socketAccept(Native Method) at java.net.AbstractPlainSocketImpl.accept(AbstractPlainSocketImpl.java:409) at java.net.ServerSocket.implAccept(ServerSocket.java:545) at java.net.ServerSocket.accept(ServerSocket.java:513) at sun.rmi.transport.tcp.TCPTransport$AcceptLoop.executeAcceptLoop(TCPTransport.java:400) at sun.rmi.transport.tcp.TCPTransport$AcceptLoop.run(TCPTransport.java:372) at java.lang.Thread.run(Thread.java:745) "Service Thread" #23 daemon prio=9 os_prio=0 tid=0x00007f138c484800 nid=0xfd440 runnable [0x0000000000000000] java.lang.Thread.State: RUNNABLE "C1 CompilerThread14" #22 daemon prio=9 os_prio=0 tid=0x00007f138c47f800 nid=0xfd43f waiting on condition [0x0000000000000000] java.lang.Thread.State: RUNNABLE "C1 CompilerThread13" #21 daemon prio=9 os_prio=0 tid=0x00007f138c47d800 nid=0xfd43e waiting on condition [0x0000000000000000] java.lang.Thread.State: RUNNABLE "C1 CompilerThread12" #20 daemon prio=9 os_prio=0 tid=0x00007f138c47b800 nid=0xfd43d waiting on condition [0x0000000000000000] java.lang.Thread.State: RUNNABLE "C1 CompilerThread11" #19 daemon prio=9 os_prio=0 tid=0x00007f138c479800 nid=0xfd43c waiting on condition [0x0000000000000000] java.lang.Thread.State: RUNNABLE "C1 CompilerThread10" #18 daemon prio=9 os_prio=0 tid=0x00007f138c477800 nid=0xfd43b waiting on condition [0x0000000000000000] java.lang.Thread.State: RUNNABLE "C2 CompilerThread9" #17 daemon prio=9 os_prio=0 tid=0x00007f138c475000 nid=0xfd43a waiting on condition [0x0000000000000000] java.lang.Thread.State: RUNNABLE "C2 CompilerThread8" #16 daemon prio=9 os_prio=0 tid=0x00007f138c472800 nid=0xfd439 waiting on condition [0x0000000000000000] java.lang.Thread.State: RUNNABLE "C2 CompilerThread7" #15 daemon prio=9 os_prio=0 tid=0x00007f138c470800 nid=0xfd438 waiting on condition [0x0000000000000000] java.lang.Thread.State: RUNNABLE "C2 CompilerThread6" #14 daemon prio=9 os_prio=0 tid=0x00007f138c46e800 nid=0xfd437 waiting on condition [0x0000000000000000] java.lang.Thread.State: RUNNABLE "C2 CompilerThread5" #13 daemon prio=9 os_prio=0 tid=0x00007f138c46c800 nid=0xfd436 waiting on condition [0x0000000000000000] java.lang.Thread.State: RUNNABLE "C2 CompilerThread4" #12 daemon prio=9 os_prio=0 tid=0x00007f138c46a800 nid=0xfd435 waiting on condition [0x0000000000000000] java.lang.Thread.State: RUNNABLE "C2 CompilerThread3" #11 daemon prio=9 os_prio=0 tid=0x00007f138c460000 nid=0xfd434 waiting on condition [0x0000000000000000] java.lang.Thread.State: RUNNABLE "C2 CompilerThread2" #10 daemon prio=9 os_prio=0 tid=0x00007f138c45e000 nid=0xfd433 waiting on condition [0x0000000000000000] java.lang.Thread.State: RUNNABLE "C2 CompilerThread1" #9 daemon prio=9 os_prio=0 tid=0x00007f138c45c000 nid=0xfd432 waiting on condition [0x0000000000000000] java.lang.Thread.State: RUNNABLE "C2 CompilerThread0" #8 daemon prio=9 os_prio=0 tid=0x00007f138c459800 nid=0xfd431 waiting on condition [0x0000000000000000] java.lang.Thread.State: RUNNABLE "JDWP Event Helper Thread" #7 daemon prio=10 os_prio=0 tid=0x00007f138c457800 nid=0xfd430 runnable [0x0000000000000000] java.lang.Thread.State: RUNNABLE "JDWP Transport Listener: dt_socket" #6 daemon prio=10 os_prio=0 tid=0x00007f138c453000 nid=0xfd42f runnable [0x0000000000000000] java.lang.Thread.State: RUNNABLE "Signal Dispatcher" #5 daemon prio=9 os_prio=0 tid=0x00007f138c445800 nid=0xfd42e runnable [0x0000000000000000] java.lang.Thread.State: RUNNABLE "Surrogate Locker Thread (Concurrent GC)" #4 daemon prio=9 os_prio=0 tid=0x00007f138c444000 nid=0xfd42d waiting on condition [0x0000000000000000] java.lang.Thread.State: RUNNABLE "Finalizer" #3 daemon prio=8 os_prio=0 tid=0x00007f138c412000 nid=0xfd42c in Object.wait() [0x00007f126eb19000] java.lang.Thread.State: WAITING (on object monitor) at java.lang.Object.wait(Native Method) at java.lang.ref.ReferenceQueue.remove(ReferenceQueue.java:143) - locked <0x00000003d80276c0> (a java.lang.ref.ReferenceQueue$Lock) at java.lang.ref.ReferenceQueue.remove(ReferenceQueue.java:164) at java.lang.ref.Finalizer$FinalizerThread.run(Finalizer.java:209) "Reference Handler" #2 daemon prio=10 os_prio=0 tid=0x00007f138c40d800 nid=0xfd42b in Object.wait() [0x00007f126ec1a000] java.lang.Thread.State: WAITING (on object monitor) at java.lang.Object.wait(Native Method) at java.lang.Object.wait(Object.java:502) at java.lang.ref.Reference.tryHandlePending(Reference.java:191) - locked <0x00000003d8025fd0> (a java.lang.ref.Reference$Lock) at java.lang.ref.Reference$ReferenceHandler.run(Reference.java:153) "main" #1 prio=5 os_prio=0 tid=0x00007f138c015800 nid=0xfd3f3 waiting on condition [0x00007f1392e66000] java.lang.Thread.State: WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00000003e7035990> (a java.util.concurrent.CountDownLatch$Sync) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer.parkAndCheckInterrupt(AbstractQueuedSynchronizer.java:836) at java.util.concurrent.locks.AbstractQueuedSynchronizer.doAcquireSharedInterruptibly(AbstractQueuedSynchronizer.java:997) at java.util.concurrent.locks.AbstractQueuedSynchronizer.acquireSharedInterruptibly(AbstractQueuedSynchronizer.java:1304) at java.util.concurrent.CountDownLatch.await(CountDownLatch.java:231) at kafka.server.KafkaServer.awaitShutdown(KafkaServer.scala:597) at kafka.server.KafkaServerStartable.awaitShutdown(KafkaServerStartable.scala:68) at kafka.Kafka$.main(Kafka.scala:68) at kafka.Kafka.main(Kafka.scala) "VM Thread" os_prio=0 tid=0x00007f138c406000 nid=0xfd42a runnable "Gang worker#0 (Parallel GC Threads)" os_prio=0 tid=0x00007f138c02e000 nid=0xfd3f4 runnable "Gang worker#1 (Parallel GC Threads)" os_prio=0 tid=0x00007f138c030000 nid=0xfd3f5 runnable "Gang worker#2 (Parallel GC Threads)" os_prio=0 tid=0x00007f138c031800 nid=0xfd3f6 runnable "Gang worker#3 (Parallel GC Threads)" os_prio=0 tid=0x00007f138c033800 nid=0xfd3f7 runnable "Gang worker#4 (Parallel GC Threads)" os_prio=0 tid=0x00007f138c035000 nid=0xfd3f8 runnable "Gang worker#5 (Parallel GC Threads)" os_prio=0 tid=0x00007f138c037000 nid=0xfd3f9 runnable "Gang worker#6 (Parallel GC Threads)" os_prio=0 tid=0x00007f138c038800 nid=0xfd3fa runnable "Gang worker#7 (Parallel GC Threads)" os_prio=0 tid=0x00007f138c03a800 nid=0xfd3fb runnable "Gang worker#8 (Parallel GC Threads)" os_prio=0 tid=0x00007f138c03c000 nid=0xfd3fc runnable "Gang worker#9 (Parallel GC Threads)" os_prio=0 tid=0x00007f138c03e000 nid=0xfd3fd runnable "Gang worker#10 (Parallel GC Threads)" os_prio=0 tid=0x00007f138c03f800 nid=0xfd3fe runnable "Gang worker#11 (Parallel GC Threads)" os_prio=0 tid=0x00007f138c041800 nid=0xfd3ff runnable "Gang worker#12 (Parallel GC Threads)" os_prio=0 tid=0x00007f138c043000 nid=0xfd400 runnable "Gang worker#13 (Parallel GC Threads)" os_prio=0 tid=0x00007f138c045000 nid=0xfd401 runnable "Gang worker#14 (Parallel GC Threads)" os_prio=0 tid=0x00007f138c046800 nid=0xfd402 runnable "Gang worker#15 (Parallel GC Threads)" os_prio=0 tid=0x00007f138c048800 nid=0xfd403 runnable "Gang worker#16 (Parallel GC Threads)" os_prio=0 tid=0x00007f138c04a000 nid=0xfd404 runnable "Gang worker#17 (Parallel GC Threads)" os_prio=0 tid=0x00007f138c04b800 nid=0xfd405 runnable "Gang worker#18 (Parallel GC Threads)" os_prio=0 tid=0x00007f138c04d800 nid=0xfd406 runnable "Gang worker#19 (Parallel GC Threads)" os_prio=0 tid=0x00007f138c04f000 nid=0xfd407 runnable "Gang worker#20 (Parallel GC Threads)" os_prio=0 tid=0x00007f138c051000 nid=0xfd408 runnable "Gang worker#21 (Parallel GC Threads)" os_prio=0 tid=0x00007f138c052800 nid=0xfd409 runnable "Gang worker#22 (Parallel GC Threads)" os_prio=0 tid=0x00007f138c054800 nid=0xfd40a runnable "G1 Main Concurrent Mark GC Thread" os_prio=0 tid=0x00007f138c095000 nid=0xfd423 runnable "Gang worker#0 (G1 Parallel Marking Threads)" os_prio=0 tid=0x00007f138c096800 nid=0xfd424 runnable "Gang worker#1 (G1 Parallel Marking Threads)" os_prio=0 tid=0x00007f138c098800 nid=0xfd425 runnable "Gang worker#2 (G1 Parallel Marking Threads)" os_prio=0 tid=0x00007f138c09a000 nid=0xfd426 runnable "Gang worker#3 (G1 Parallel Marking Threads)" os_prio=0 tid=0x00007f138c09c000 nid=0xfd427 runnable "Gang worker#4 (G1 Parallel Marking Threads)" os_prio=0 tid=0x00007f138c09d800 nid=0xfd428 runnable "Gang worker#5 (G1 Parallel Marking Threads)" os_prio=0 tid=0x00007f138c09f800 nid=0xfd429 runnable "G1 Concurrent Refinement Thread#0" os_prio=0 tid=0x00007f138c081000 nid=0xfd422 runnable "G1 Concurrent Refinement Thread#1" os_prio=0 tid=0x00007f138c07f000 nid=0xfd421 runnable "G1 Concurrent Refinement Thread#2" os_prio=0 tid=0x00007f138c07d800 nid=0xfd420 runnable "G1 Concurrent Refinement Thread#3" os_prio=0 tid=0x00007f138c07b800 nid=0xfd41f runnable "G1 Concurrent Refinement Thread#4" os_prio=0 tid=0x00007f138c079800 nid=0xfd41e runnable "G1 Concurrent Refinement Thread#5" os_prio=0 tid=0x00007f138c078000 nid=0xfd41d runnable "G1 Concurrent Refinement Thread#6" os_prio=0 tid=0x00007f138c076000 nid=0xfd41c runnable "G1 Concurrent Refinement Thread#7" os_prio=0 tid=0x00007f138c074800 nid=0xfd41b runnable "G1 Concurrent Refinement Thread#8" os_prio=0 tid=0x00007f138c072800 nid=0xfd41a runnable "G1 Concurrent Refinement Thread#9" os_prio=0 tid=0x00007f138c071000 nid=0xfd419 runnable "G1 Concurrent Refinement Thread#10" os_prio=0 tid=0x00007f138c06f000 nid=0xfd418 runnable "G1 Concurrent Refinement Thread#11" os_prio=0 tid=0x00007f138c06d800 nid=0xfd417 runnable "G1 Concurrent Refinement Thread#12" os_prio=0 tid=0x00007f138c06b800 nid=0xfd416 runnable "G1 Concurrent Refinement Thread#13" os_prio=0 tid=0x00007f138c06a000 nid=0xfd415 runnable "G1 Concurrent Refinement Thread#14" os_prio=0 tid=0x00007f138c068000 nid=0xfd414 runnable "G1 Concurrent Refinement Thread#15" os_prio=0 tid=0x00007f138c066000 nid=0xfd413 runnable "G1 Concurrent Refinement Thread#16" os_prio=0 tid=0x00007f138c064800 nid=0xfd412 runnable "G1 Concurrent Refinement Thread#17" os_prio=0 tid=0x00007f138c062800 nid=0xfd411 runnable "G1 Concurrent Refinement Thread#18" os_prio=0 tid=0x00007f138c061000 nid=0xfd410 runnable "G1 Concurrent Refinement Thread#19" os_prio=0 tid=0x00007f138c05f000 nid=0xfd40f runnable "G1 Concurrent Refinement Thread#20" os_prio=0 tid=0x00007f138c05d800 nid=0xfd40e runnable "G1 Concurrent Refinement Thread#21" os_prio=0 tid=0x00007f138c05b800 nid=0xfd40d runnable "G1 Concurrent Refinement Thread#22" os_prio=0 tid=0x00007f138c05a000 nid=0xfd40c runnable "G1 Concurrent Refinement Thread#23" os_prio=0 tid=0x00007f138c058000 nid=0xfd40b runnable "VM Periodic Task Thread" os_prio=0 tid=0x00007f138c7ca000 nid=0xfd445 waiting on condition JNI global references: 4207