2016-08-18 12:51:56
Full thread dump Java HotSpot(TM) 64-Bit Server VM (25.92-b14 mixed mode):

"Attach Listener" #119 daemon prio=9 os_prio=0 tid=0x00007f0d78001000 nid=0xa2305 waiting on condition [0x0000000000000000]
   java.lang.Thread.State: RUNNABLE

"Thread-2" #29 prio=5 os_prio=0 tid=0x00007ef84c00a800 nid=0xa1fa6 waiting on condition [0x00007f0bca3f1000]
   java.lang.Thread.State: WAITING (parking)
	at sun.misc.Unsafe.park(Native Method)
	- parking to wait for  <0x00000006c54f3b48> (a java.util.concurrent.CountDownLatch$Sync)
	at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)
	at java.util.concurrent.locks.AbstractQueuedSynchronizer.parkAndCheckInterrupt(AbstractQueuedSynchronizer.java:836)
	at java.util.concurrent.locks.AbstractQueuedSynchronizer.doAcquireSharedInterruptibly(AbstractQueuedSynchronizer.java:997)
	at java.util.concurrent.locks.AbstractQueuedSynchronizer.acquireSharedInterruptibly(AbstractQueuedSynchronizer.java:1304)
	at java.util.concurrent.CountDownLatch.await(CountDownLatch.java:231)
	at kafka.utils.ShutdownableThread.awaitShutdown(ShutdownableThread.scala:52)
	at kafka.server.AbstractFetcherThread.shutdown(AbstractFetcherThread.scala:79)
	at kafka.server.ReplicaFetcherThread.shutdown(ReplicaFetcherThread.scala:111)
	at kafka.server.AbstractFetcherManager$$anonfun$closeAllFetchers$2.apply(AbstractFetcherManager.scala:123)
	at kafka.server.AbstractFetcherManager$$anonfun$closeAllFetchers$2.apply(AbstractFetcherManager.scala:122)
	at scala.collection.TraversableLike$WithFilter$$anonfun$foreach$1.apply(TraversableLike.scala:778)
	at scala.collection.mutable.HashMap$$anonfun$foreach$1.apply(HashMap.scala:99)
	at scala.collection.mutable.HashMap$$anonfun$foreach$1.apply(HashMap.scala:99)
	at scala.collection.mutable.HashTable$class.foreachEntry(HashTable.scala:230)
	at scala.collection.mutable.HashMap.foreachEntry(HashMap.scala:40)
	at scala.collection.mutable.HashMap.foreach(HashMap.scala:99)
	at scala.collection.TraversableLike$WithFilter.foreach(TraversableLike.scala:777)
	at kafka.server.AbstractFetcherManager.closeAllFetchers(AbstractFetcherManager.scala:122)
	- locked <0x00000003e8bdf750> (a java.lang.Object)
	at kafka.server.ReplicaFetcherManager.shutdown(ReplicaFetcherManager.scala:41)
	at kafka.server.ReplicaManager.shutdown(ReplicaManager.scala:901)
	at kafka.server.KafkaServer$$anonfun$shutdown$7.apply$mcV$sp(KafkaServer.scala:581)
	at kafka.utils.CoreUtils$.swallow(CoreUtils.scala:76)
	at kafka.utils.Logging$class.swallowWarn(Logging.scala:92)
	at kafka.utils.CoreUtils$.swallowWarn(CoreUtils.scala:47)
	at kafka.utils.Logging$class.swallow(Logging.scala:94)
	at kafka.utils.CoreUtils$.swallow(CoreUtils.scala:47)
	at kafka.server.KafkaServer.shutdown(KafkaServer.scala:581)
	at kafka.server.KafkaServerStartable.shutdown(KafkaServerStartable.scala:49)
	at kafka.Kafka$$anon$1.run(Kafka.scala:63)

"ReplicaFetcherThread-2-3" #118 prio=5 os_prio=0 tid=0x00007ef8dc187800 nid=0xa1f7d waiting for monitor entry [0x00007f0bca4f3000]
   java.lang.Thread.State: BLOCKED (on object monitor)
	at java.lang.Shutdown.exit(Shutdown.java:212)
	- waiting to lock <0x00000003d81e16a8> (a java.lang.Class for java.lang.Shutdown)
	at java.lang.Runtime.exit(Runtime.java:109)
	at java.lang.System.exit(System.java:971)
	at org.apache.kafka.common.errors.FatalExitError.shutdownSystem(FatalExitError.java:61)
	at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:69)

"ReplicaFetcherThread-1-3" #117 prio=5 os_prio=0 tid=0x00007ef8dc16d000 nid=0xa1f7c waiting for monitor entry [0x00007f0bca5f4000]
   java.lang.Thread.State: BLOCKED (on object monitor)
	at java.lang.Shutdown.exit(Shutdown.java:212)
	- waiting to lock <0x00000003d81e16a8> (a java.lang.Class for java.lang.Shutdown)
	at java.lang.Runtime.exit(Runtime.java:109)
	at java.lang.System.exit(System.java:971)
	at org.apache.kafka.common.errors.FatalExitError.shutdownSystem(FatalExitError.java:61)
	at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:69)

"ReplicaFetcherThread-3-2" #116 prio=5 os_prio=0 tid=0x00007ef8dc152800 nid=0xa1f7a waiting for monitor entry [0x00007f0bca6f5000]
   java.lang.Thread.State: BLOCKED (on object monitor)
	at java.lang.Shutdown.exit(Shutdown.java:212)
	- waiting to lock <0x00000003d81e16a8> (a java.lang.Class for java.lang.Shutdown)
	at java.lang.Runtime.exit(Runtime.java:109)
	at java.lang.System.exit(System.java:971)
	at org.apache.kafka.common.errors.FatalExitError.shutdownSystem(FatalExitError.java:61)
	at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:69)

"ReplicaFetcherThread-0-3" #111 prio=5 os_prio=0 tid=0x00007ef8dc0cc800 nid=0xa1f74 waiting for monitor entry [0x00007f0bcabfa000]
   java.lang.Thread.State: BLOCKED (on object monitor)
	at java.lang.Shutdown.exit(Shutdown.java:212)
	- waiting to lock <0x00000003d81e16a8> (a java.lang.Class for java.lang.Shutdown)
	at java.lang.Runtime.exit(Runtime.java:109)
	at java.lang.System.exit(System.java:971)
	at org.apache.kafka.common.errors.FatalExitError.shutdownSystem(FatalExitError.java:61)
	at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:69)

"ReplicaFetcherThread-2-2" #110 prio=5 os_prio=0 tid=0x00007ef8dc0b2000 nid=0xa1f73 waiting for monitor entry [0x00007f0bccbf4000]
   java.lang.Thread.State: BLOCKED (on object monitor)
	at java.lang.Shutdown.exit(Shutdown.java:212)
	- waiting to lock <0x00000003d81e16a8> (a java.lang.Class for java.lang.Shutdown)
	at java.lang.Runtime.exit(Runtime.java:109)
	at java.lang.System.exit(System.java:971)
	at org.apache.kafka.common.errors.FatalExitError.shutdownSystem(FatalExitError.java:61)
	at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:69)

"ReplicaFetcherThread-0-2" #108 prio=5 os_prio=0 tid=0x00007ef8dc062800 nid=0xa1f71 in Object.wait() [0x00007f0bccdf6000]
   java.lang.Thread.State: WAITING (on object monitor)
	at java.lang.Object.wait(Native Method)
	at java.lang.Thread.join(Thread.java:1245)
	- locked <0x00000003d822fd08> (a kafka.Kafka$$anon$1)
	at java.lang.Thread.join(Thread.java:1319)
	at java.lang.ApplicationShutdownHooks.runHooks(ApplicationShutdownHooks.java:106)
	at java.lang.ApplicationShutdownHooks$1.run(ApplicationShutdownHooks.java:46)
	at java.lang.Shutdown.runHooks(Shutdown.java:123)
	at java.lang.Shutdown.sequence(Shutdown.java:167)
	at java.lang.Shutdown.exit(Shutdown.java:212)
	- locked <0x00000003d81e16a8> (a java.lang.Class for java.lang.Shutdown)
	at java.lang.Runtime.exit(Runtime.java:109)
	at java.lang.System.exit(System.java:971)
	at org.apache.kafka.common.errors.FatalExitError.shutdownSystem(FatalExitError.java:61)
	at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:69)

"ReplicaFetcherThread-3-3" #107 prio=5 os_prio=0 tid=0x00007ef8dc061800 nid=0xa1f70 waiting for monitor entry [0x00007f0bccef7000]
   java.lang.Thread.State: BLOCKED (on object monitor)
	at java.lang.Shutdown.exit(Shutdown.java:212)
	- waiting to lock <0x00000003d81e16a8> (a java.lang.Class for java.lang.Shutdown)
	at java.lang.Runtime.exit(Runtime.java:109)
	at java.lang.System.exit(System.java:971)
	at org.apache.kafka.common.errors.FatalExitError.shutdownSystem(FatalExitError.java:61)
	at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:69)

"JMX server connection timeout 103" #103 daemon prio=5 os_prio=0 tid=0x00007f0d00006000 nid=0xa1cf1 in Object.wait() [0x00007f0d88546000]
   java.lang.Thread.State: TIMED_WAITING (on object monitor)
	at java.lang.Object.wait(Native Method)
	at com.sun.jmx.remote.internal.ServerCommunicatorAdmin$Timeout.run(ServerCommunicatorAdmin.java:168)
	- locked <0x00000003e8ebfcc0> (a [I)
	at java.lang.Thread.run(Thread.java:745)

"group-metadata-manager-0" #68 daemon prio=5 os_prio=0 tid=0x00007f0ea8ddc000 nid=0xa1ccd waiting on condition [0x00007f0bd57f2000]
   java.lang.Thread.State: TIMED_WAITING (parking)
	at sun.misc.Unsafe.park(Native Method)
	- parking to wait for  <0x00000003e8ddd320> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject)
	at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
	at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)
	at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1093)
	at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809)
	at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1067)
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127)
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
	at java.lang.Thread.run(Thread.java:745)

"ExpirationReaper-5" #67 prio=5 os_prio=0 tid=0x00007f0ea8dd4800 nid=0xa1ccb waiting on condition [0x00007f0bd58f3000]
   java.lang.Thread.State: TIMED_WAITING (parking)
	at sun.misc.Unsafe.park(Native Method)
	- parking to wait for  <0x00000003e8b41d10> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject)
	at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
	at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)
	at java.util.concurrent.DelayQueue.poll(DelayQueue.java:259)
	at kafka.utils.timer.SystemTimer.advanceClock(Timer.scala:105)
	at kafka.server.DelayedOperationPurgatory.advanceClock(DelayedOperation.scala:350)
	at kafka.server.DelayedOperationPurgatory$ExpiredOperationReaper.doWork(DelayedOperation.scala:374)
	at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:65)

"ExpirationReaper-5" #66 prio=5 os_prio=0 tid=0x00007f0ea8dd3800 nid=0xa1cc9 waiting on condition [0x00007f0bd59f4000]
   java.lang.Thread.State: TIMED_WAITING (parking)
	at sun.misc.Unsafe.park(Native Method)
	- parking to wait for  <0x00000003e8b16600> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject)
	at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
	at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)
	at java.util.concurrent.DelayQueue.poll(DelayQueue.java:259)
	at kafka.utils.timer.SystemTimer.advanceClock(Timer.scala:105)
	at kafka.server.DelayedOperationPurgatory.advanceClock(DelayedOperation.scala:350)
	at kafka.server.DelayedOperationPurgatory$ExpiredOperationReaper.doWork(DelayedOperation.scala:374)
	at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:65)

"metrics-meter-tick-thread-2" #61 daemon prio=5 os_prio=0 tid=0x00007f0ea8c7e800 nid=0xa1cc4 waiting on condition [0x00007f0bd5ef9000]
   java.lang.Thread.State: TIMED_WAITING (parking)
	at sun.misc.Unsafe.park(Native Method)
	- parking to wait for  <0x00000003e8b16830> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject)
	at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
	at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)
	at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1093)
	at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809)
	at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1067)
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127)
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
	at java.lang.Thread.run(Thread.java:745)

"ExpirationReaper-5" #60 prio=5 os_prio=0 tid=0x00007f0ea8c7d800 nid=0xa1cc3 waiting on condition [0x00007f0bd5ffa000]
   java.lang.Thread.State: TIMED_WAITING (parking)
	at sun.misc.Unsafe.park(Native Method)
	- parking to wait for  <0x00000003e8b42460> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject)
	at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
	at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)
	at java.util.concurrent.DelayQueue.poll(DelayQueue.java:259)
	at kafka.utils.timer.SystemTimer.advanceClock(Timer.scala:105)
	at kafka.server.DelayedOperationPurgatory.advanceClock(DelayedOperation.scala:350)
	at kafka.server.DelayedOperationPurgatory$ExpiredOperationReaper.doWork(DelayedOperation.scala:374)
	at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:65)

"ExpirationReaper-5" #59 prio=5 os_prio=0 tid=0x00007f0ea8c7d000 nid=0xa1cc2 waiting on condition [0x00007f0bd89f2000]
   java.lang.Thread.State: TIMED_WAITING (parking)
	at sun.misc.Unsafe.park(Native Method)
	- parking to wait for  <0x00000003e8b426b8> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject)
	at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
	at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)
	at java.util.concurrent.DelayQueue.poll(DelayQueue.java:259)
	at kafka.utils.timer.SystemTimer.advanceClock(Timer.scala:105)
	at kafka.server.DelayedOperationPurgatory.advanceClock(DelayedOperation.scala:350)
	at kafka.server.DelayedOperationPurgatory$ExpiredOperationReaper.doWork(DelayedOperation.scala:374)
	at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:65)

"metrics-meter-tick-thread-1" #45 daemon prio=5 os_prio=0 tid=0x00007f0ea84c6000 nid=0xa1cb3 waiting on condition [0x00007f0d2812a000]
   java.lang.Thread.State: WAITING (parking)
	at sun.misc.Unsafe.park(Native Method)
	- parking to wait for  <0x00000003e8b16830> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject)
	at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)
	at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039)
	at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1088)
	at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809)
	at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1067)
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127)
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
	at java.lang.Thread.run(Thread.java:745)

"RMI TCP Connection(13)-5.45.227.235" #39 daemon prio=5 os_prio=0 tid=0x00007f0d24003800 nid=0xa18ab runnable [0x00007f0d88646000]
   java.lang.Thread.State: RUNNABLE
	at java.net.SocketInputStream.socketRead0(Native Method)
	at java.net.SocketInputStream.socketRead(SocketInputStream.java:116)
	at java.net.SocketInputStream.read(SocketInputStream.java:170)
	at java.net.SocketInputStream.read(SocketInputStream.java:141)
	at java.io.BufferedInputStream.fill(BufferedInputStream.java:246)
	at java.io.BufferedInputStream.read(BufferedInputStream.java:265)
	- locked <0x00000006deca0c78> (a java.io.BufferedInputStream)
	at java.io.FilterInputStream.read(FilterInputStream.java:83)
	at sun.rmi.transport.tcp.TCPTransport.handleMessages(TCPTransport.java:550)
	at sun.rmi.transport.tcp.TCPTransport$ConnectionHandler.run0(TCPTransport.java:826)
	at sun.rmi.transport.tcp.TCPTransport$ConnectionHandler.lambda$run$0(TCPTransport.java:683)
	at sun.rmi.transport.tcp.TCPTransport$ConnectionHandler$$Lambda$2/1126075275.run(Unknown Source)
	at java.security.AccessController.doPrivileged(Native Method)
	at sun.rmi.transport.tcp.TCPTransport$ConnectionHandler.run(TCPTransport.java:682)
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142)
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
	at java.lang.Thread.run(Thread.java:745)

"RMI Scheduler(0)" #38 daemon prio=5 os_prio=0 tid=0x00007f0d1003f000 nid=0xa18a9 waiting on condition [0x00007f0d88748000]
   java.lang.Thread.State: TIMED_WAITING (parking)
	at sun.misc.Unsafe.park(Native Method)
	- parking to wait for  <0x00000003d8005170> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject)
	at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
	at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)
	at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1093)
	at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809)
	at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1067)
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127)
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
	at java.lang.Thread.run(Thread.java:745)

"main-EventThread" #36 daemon prio=5 os_prio=0 tid=0x00007f0ea8b0e800 nid=0xa18a5 waiting on condition [0x00007f0d88849000]
   java.lang.Thread.State: WAITING (parking)
	at sun.misc.Unsafe.park(Native Method)
	- parking to wait for  <0x00000003d80067a8> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject)
	at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)
	at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039)
	at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442)
	at org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:494)

"main-SendThread(zk03ep.stat.yandex.net:2181)" #35 daemon prio=5 os_prio=0 tid=0x00007f0ea8ad2800 nid=0xa18a4 runnable [0x00007f0d88b4a000]
   java.lang.Thread.State: RUNNABLE
	at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method)
	at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269)
	at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93)
	at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86)
	- locked <0x00000003d8005558> (a sun.nio.ch.Util$2)
	- locked <0x00000003d8005548> (a java.util.Collections$UnmodifiableSet)
	- locked <0x00000003d8005500> (a sun.nio.ch.EPollSelectorImpl)
	at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97)
	at org.apache.zookeeper.ClientCnxnSocketNIO.doTransport(ClientCnxnSocketNIO.java:349)
	at org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1081)

"ZkClient-EventThread-34-zk01ep.stat.yandex.net:2181,zk02ep.stat.yandex.net:2181,zk03ep.stat.yandex.net:2181/kafka-iva-et" #34 daemon prio=5 os_prio=0 tid=0x00007f0ea8ab8800 nid=0xa18a3 waiting on condition [0x00007f0d88d4c000]
   java.lang.Thread.State: WAITING (parking)
	at sun.misc.Unsafe.park(Native Method)
	- parking to wait for  <0x00000003d80069b0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject)
	at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)
	at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039)
	at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442)
	at org.I0Itec.zkclient.ZkEventThread.run(ZkEventThread.java:67)

"SensorExpiryThread" #30 daemon prio=5 os_prio=0 tid=0x00007f0ea8aa6000 nid=0xa189a waiting on condition [0x00007f0d88e4d000]
   java.lang.Thread.State: TIMED_WAITING (parking)
	at sun.misc.Unsafe.park(Native Method)
	- parking to wait for  <0x00000003d8006bb0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject)
	at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
	at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)
	at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1093)
	at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809)
	at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1067)
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127)
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
	at java.lang.Thread.run(Thread.java:745)

"RMI TCP Accept-0" #27 daemon prio=5 os_prio=0 tid=0x00007f0ea8809000 nid=0xa1894 runnable [0x00007f0d8944f000]
   java.lang.Thread.State: RUNNABLE
	at java.net.PlainSocketImpl.socketAccept(Native Method)
	at java.net.AbstractPlainSocketImpl.accept(AbstractPlainSocketImpl.java:409)
	at java.net.ServerSocket.implAccept(ServerSocket.java:545)
	at java.net.ServerSocket.accept(ServerSocket.java:513)
	at sun.management.jmxremote.LocalRMIServerSocketFactory$1.accept(LocalRMIServerSocketFactory.java:52)
	at sun.rmi.transport.tcp.TCPTransport$AcceptLoop.executeAcceptLoop(TCPTransport.java:400)
	at sun.rmi.transport.tcp.TCPTransport$AcceptLoop.run(TCPTransport.java:372)
	at java.lang.Thread.run(Thread.java:745)

"RMI TCP Accept-9999" #26 daemon prio=5 os_prio=0 tid=0x00007f0ea87ff800 nid=0xa1893 runnable [0x00007f0d89550000]
   java.lang.Thread.State: RUNNABLE
	at java.net.PlainSocketImpl.socketAccept(Native Method)
	at java.net.AbstractPlainSocketImpl.accept(AbstractPlainSocketImpl.java:409)
	at java.net.ServerSocket.implAccept(ServerSocket.java:545)
	at java.net.ServerSocket.accept(ServerSocket.java:513)
	at sun.rmi.transport.tcp.TCPTransport$AcceptLoop.executeAcceptLoop(TCPTransport.java:400)
	at sun.rmi.transport.tcp.TCPTransport$AcceptLoop.run(TCPTransport.java:372)
	at java.lang.Thread.run(Thread.java:745)

"RMI TCP Accept-0" #25 daemon prio=5 os_prio=0 tid=0x00007f0ea87e9800 nid=0xa1892 runnable [0x00007f0d89651000]
   java.lang.Thread.State: RUNNABLE
	at java.net.PlainSocketImpl.socketAccept(Native Method)
	at java.net.AbstractPlainSocketImpl.accept(AbstractPlainSocketImpl.java:409)
	at java.net.ServerSocket.implAccept(ServerSocket.java:545)
	at java.net.ServerSocket.accept(ServerSocket.java:513)
	at sun.rmi.transport.tcp.TCPTransport$AcceptLoop.executeAcceptLoop(TCPTransport.java:400)
	at sun.rmi.transport.tcp.TCPTransport$AcceptLoop.run(TCPTransport.java:372)
	at java.lang.Thread.run(Thread.java:745)

"Service Thread" #23 daemon prio=9 os_prio=0 tid=0x00007f0ea8485800 nid=0xa1891 runnable [0x0000000000000000]
   java.lang.Thread.State: RUNNABLE

"C1 CompilerThread14" #22 daemon prio=9 os_prio=0 tid=0x00007f0ea8480000 nid=0xa1890 waiting on condition [0x0000000000000000]
   java.lang.Thread.State: RUNNABLE

"C1 CompilerThread13" #21 daemon prio=9 os_prio=0 tid=0x00007f0ea847e000 nid=0xa188f waiting on condition [0x0000000000000000]
   java.lang.Thread.State: RUNNABLE

"C1 CompilerThread12" #20 daemon prio=9 os_prio=0 tid=0x00007f0ea847c000 nid=0xa188e waiting on condition [0x0000000000000000]
   java.lang.Thread.State: RUNNABLE

"C1 CompilerThread11" #19 daemon prio=9 os_prio=0 tid=0x00007f0ea8479800 nid=0xa188d waiting on condition [0x0000000000000000]
   java.lang.Thread.State: RUNNABLE

"C1 CompilerThread10" #18 daemon prio=9 os_prio=0 tid=0x00007f0ea8478000 nid=0xa188c waiting on condition [0x0000000000000000]
   java.lang.Thread.State: RUNNABLE

"C2 CompilerThread9" #17 daemon prio=9 os_prio=0 tid=0x00007f0ea8475000 nid=0xa188b waiting on condition [0x0000000000000000]
   java.lang.Thread.State: RUNNABLE

"C2 CompilerThread8" #16 daemon prio=9 os_prio=0 tid=0x00007f0ea8473000 nid=0xa188a waiting on condition [0x0000000000000000]
   java.lang.Thread.State: RUNNABLE

"C2 CompilerThread7" #15 daemon prio=9 os_prio=0 tid=0x00007f0ea8471000 nid=0xa1889 waiting on condition [0x0000000000000000]
   java.lang.Thread.State: RUNNABLE

"C2 CompilerThread6" #14 daemon prio=9 os_prio=0 tid=0x00007f0ea846f000 nid=0xa1888 waiting on condition [0x0000000000000000]
   java.lang.Thread.State: RUNNABLE

"C2 CompilerThread5" #13 daemon prio=9 os_prio=0 tid=0x00007f0ea846c800 nid=0xa1887 waiting on condition [0x0000000000000000]
   java.lang.Thread.State: RUNNABLE

"C2 CompilerThread4" #12 daemon prio=9 os_prio=0 tid=0x00007f0ea846a800 nid=0xa1886 waiting on condition [0x0000000000000000]
   java.lang.Thread.State: RUNNABLE

"C2 CompilerThread3" #11 daemon prio=9 os_prio=0 tid=0x00007f0ea8460800 nid=0xa1885 waiting on condition [0x0000000000000000]
   java.lang.Thread.State: RUNNABLE

"C2 CompilerThread2" #10 daemon prio=9 os_prio=0 tid=0x00007f0ea845e000 nid=0xa1884 waiting on condition [0x0000000000000000]
   java.lang.Thread.State: RUNNABLE

"C2 CompilerThread1" #9 daemon prio=9 os_prio=0 tid=0x00007f0ea845c000 nid=0xa1883 waiting on condition [0x0000000000000000]
   java.lang.Thread.State: RUNNABLE

"C2 CompilerThread0" #8 daemon prio=9 os_prio=0 tid=0x00007f0ea845a000 nid=0xa1882 runnable [0x0000000000000000]
   java.lang.Thread.State: RUNNABLE

"JDWP Event Helper Thread" #7 daemon prio=10 os_prio=0 tid=0x00007f0ea8457800 nid=0xa1881 runnable [0x0000000000000000]
   java.lang.Thread.State: RUNNABLE

"JDWP Transport Listener: dt_socket" #6 daemon prio=10 os_prio=0 tid=0x00007f0ea8453000 nid=0xa1880 runnable [0x0000000000000000]
   java.lang.Thread.State: RUNNABLE

"Signal Dispatcher" #5 daemon prio=9 os_prio=0 tid=0x00007f0ea8446000 nid=0xa187f runnable [0x0000000000000000]
   java.lang.Thread.State: RUNNABLE

"Surrogate Locker Thread (Concurrent GC)" #4 daemon prio=9 os_prio=0 tid=0x00007f0ea8444800 nid=0xa187e waiting on condition [0x0000000000000000]
   java.lang.Thread.State: RUNNABLE

"Finalizer" #3 daemon prio=8 os_prio=0 tid=0x00007f0ea8412800 nid=0xa187d in Object.wait() [0x00007f0d8b924000]
   java.lang.Thread.State: WAITING (on object monitor)
	at java.lang.Object.wait(Native Method)
	at java.lang.ref.ReferenceQueue.remove(ReferenceQueue.java:143)
	- locked <0x00000003d8020970> (a java.lang.ref.ReferenceQueue$Lock)
	at java.lang.ref.ReferenceQueue.remove(ReferenceQueue.java:164)
	at java.lang.ref.Finalizer$FinalizerThread.run(Finalizer.java:209)

"Reference Handler" #2 daemon prio=10 os_prio=0 tid=0x00007f0ea840e000 nid=0xa187c in Object.wait() [0x00007f0d8ba25000]
   java.lang.Thread.State: WAITING (on object monitor)
	at java.lang.Object.wait(Native Method)
	at java.lang.Object.wait(Object.java:502)
	at java.lang.ref.Reference.tryHandlePending(Reference.java:191)
	- locked <0x00000003d801d1d0> (a java.lang.ref.Reference$Lock)
	at java.lang.ref.Reference$ReferenceHandler.run(Reference.java:153)

"main" #1 prio=5 os_prio=0 tid=0x00007f0ea8015800 nid=0xa1843 waiting on condition [0x00007f0eafd5e000]
   java.lang.Thread.State: WAITING (parking)
	at sun.misc.Unsafe.park(Native Method)
	- parking to wait for  <0x00000003e8ee46d8> (a java.util.concurrent.CountDownLatch$Sync)
	at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)
	at java.util.concurrent.locks.AbstractQueuedSynchronizer.parkAndCheckInterrupt(AbstractQueuedSynchronizer.java:836)
	at java.util.concurrent.locks.AbstractQueuedSynchronizer.doAcquireSharedInterruptibly(AbstractQueuedSynchronizer.java:997)
	at java.util.concurrent.locks.AbstractQueuedSynchronizer.acquireSharedInterruptibly(AbstractQueuedSynchronizer.java:1304)
	at java.util.concurrent.CountDownLatch.await(CountDownLatch.java:231)
	at kafka.server.KafkaServer.awaitShutdown(KafkaServer.scala:613)
	at kafka.server.KafkaServerStartable.awaitShutdown(KafkaServerStartable.scala:68)
	at kafka.Kafka$.main(Kafka.scala:68)
	at kafka.Kafka.main(Kafka.scala)

"VM Thread" os_prio=0 tid=0x00007f0ea8406000 nid=0xa187b runnable 

"Gang worker#0 (Parallel GC Threads)" os_prio=0 tid=0x00007f0ea802e000 nid=0xa1844 runnable 

"Gang worker#1 (Parallel GC Threads)" os_prio=0 tid=0x00007f0ea8030000 nid=0xa1845 runnable 

"Gang worker#2 (Parallel GC Threads)" os_prio=0 tid=0x00007f0ea8031800 nid=0xa1846 runnable 

"Gang worker#3 (Parallel GC Threads)" os_prio=0 tid=0x00007f0ea8033800 nid=0xa1847 runnable 

"Gang worker#4 (Parallel GC Threads)" os_prio=0 tid=0x00007f0ea8035000 nid=0xa1848 runnable 

"Gang worker#5 (Parallel GC Threads)" os_prio=0 tid=0x00007f0ea8037000 nid=0xa1849 runnable 

"Gang worker#6 (Parallel GC Threads)" os_prio=0 tid=0x00007f0ea8038800 nid=0xa184a runnable 

"Gang worker#7 (Parallel GC Threads)" os_prio=0 tid=0x00007f0ea803a800 nid=0xa184b runnable 

"Gang worker#8 (Parallel GC Threads)" os_prio=0 tid=0x00007f0ea803c000 nid=0xa184c runnable 

"Gang worker#9 (Parallel GC Threads)" os_prio=0 tid=0x00007f0ea803e000 nid=0xa184d runnable 

"Gang worker#10 (Parallel GC Threads)" os_prio=0 tid=0x00007f0ea803f800 nid=0xa184e runnable 

"Gang worker#11 (Parallel GC Threads)" os_prio=0 tid=0x00007f0ea8041800 nid=0xa184f runnable 

"Gang worker#12 (Parallel GC Threads)" os_prio=0 tid=0x00007f0ea8043000 nid=0xa1850 runnable 

"Gang worker#13 (Parallel GC Threads)" os_prio=0 tid=0x00007f0ea8045000 nid=0xa1851 runnable 

"Gang worker#14 (Parallel GC Threads)" os_prio=0 tid=0x00007f0ea8046800 nid=0xa1852 runnable 

"Gang worker#15 (Parallel GC Threads)" os_prio=0 tid=0x00007f0ea8048800 nid=0xa1853 runnable 

"Gang worker#16 (Parallel GC Threads)" os_prio=0 tid=0x00007f0ea804a000 nid=0xa1854 runnable 

"Gang worker#17 (Parallel GC Threads)" os_prio=0 tid=0x00007f0ea804c000 nid=0xa1855 runnable 

"Gang worker#18 (Parallel GC Threads)" os_prio=0 tid=0x00007f0ea804d800 nid=0xa1856 runnable 

"Gang worker#19 (Parallel GC Threads)" os_prio=0 tid=0x00007f0ea804f800 nid=0xa1857 runnable 

"Gang worker#20 (Parallel GC Threads)" os_prio=0 tid=0x00007f0ea8051000 nid=0xa1858 runnable 

"Gang worker#21 (Parallel GC Threads)" os_prio=0 tid=0x00007f0ea8053000 nid=0xa1859 runnable 

"Gang worker#22 (Parallel GC Threads)" os_prio=0 tid=0x00007f0ea8054800 nid=0xa185a runnable 

"G1 Main Concurrent Mark GC Thread" os_prio=0 tid=0x00007f0ea8095000 nid=0xa1873 runnable 

"Gang worker#0 (G1 Parallel Marking Threads)" os_prio=0 tid=0x00007f0ea8097000 nid=0xa1874 runnable 

"Gang worker#1 (G1 Parallel Marking Threads)" os_prio=0 tid=0x00007f0ea8098800 nid=0xa1875 runnable 

"Gang worker#2 (G1 Parallel Marking Threads)" os_prio=0 tid=0x00007f0ea809a800 nid=0xa1876 runnable 

"Gang worker#3 (G1 Parallel Marking Threads)" os_prio=0 tid=0x00007f0ea809c000 nid=0xa1877 runnable 

"Gang worker#4 (G1 Parallel Marking Threads)" os_prio=0 tid=0x00007f0ea809e000 nid=0xa1878 runnable 

"Gang worker#5 (G1 Parallel Marking Threads)" os_prio=0 tid=0x00007f0ea809f800 nid=0xa1879 runnable 

"G1 Concurrent Refinement Thread#0" os_prio=0 tid=0x00007f0ea8081000 nid=0xa1872 runnable 

"G1 Concurrent Refinement Thread#1" os_prio=0 tid=0x00007f0ea807f000 nid=0xa1871 runnable 

"G1 Concurrent Refinement Thread#2" os_prio=0 tid=0x00007f0ea807d800 nid=0xa1870 runnable 

"G1 Concurrent Refinement Thread#3" os_prio=0 tid=0x00007f0ea807b800 nid=0xa186f runnable 

"G1 Concurrent Refinement Thread#4" os_prio=0 tid=0x00007f0ea807a000 nid=0xa186e runnable 

"G1 Concurrent Refinement Thread#5" os_prio=0 tid=0x00007f0ea8078000 nid=0xa186d runnable 

"G1 Concurrent Refinement Thread#6" os_prio=0 tid=0x00007f0ea8076800 nid=0xa186c runnable 

"G1 Concurrent Refinement Thread#7" os_prio=0 tid=0x00007f0ea8074800 nid=0xa186b runnable 

"G1 Concurrent Refinement Thread#8" os_prio=0 tid=0x00007f0ea8073000 nid=0xa186a runnable 

"G1 Concurrent Refinement Thread#9" os_prio=0 tid=0x00007f0ea8071000 nid=0xa1869 runnable 

"G1 Concurrent Refinement Thread#10" os_prio=0 tid=0x00007f0ea806f000 nid=0xa1868 runnable 

"G1 Concurrent Refinement Thread#11" os_prio=0 tid=0x00007f0ea806d800 nid=0xa1867 runnable 

"G1 Concurrent Refinement Thread#12" os_prio=0 tid=0x00007f0ea806b800 nid=0xa1866 runnable 

"G1 Concurrent Refinement Thread#13" os_prio=0 tid=0x00007f0ea806a000 nid=0xa1865 runnable 

"G1 Concurrent Refinement Thread#14" os_prio=0 tid=0x00007f0ea8068000 nid=0xa1864 runnable 

"G1 Concurrent Refinement Thread#15" os_prio=0 tid=0x00007f0ea8066800 nid=0xa1863 runnable 

"G1 Concurrent Refinement Thread#16" os_prio=0 tid=0x00007f0ea8064800 nid=0xa1862 runnable 

"G1 Concurrent Refinement Thread#17" os_prio=0 tid=0x00007f0ea8063000 nid=0xa1861 runnable 

"G1 Concurrent Refinement Thread#18" os_prio=0 tid=0x00007f0ea8061000 nid=0xa1860 runnable 

"G1 Concurrent Refinement Thread#19" os_prio=0 tid=0x00007f0ea805f800 nid=0xa185f runnable 

"G1 Concurrent Refinement Thread#20" os_prio=0 tid=0x00007f0ea805d800 nid=0xa185e runnable 

"G1 Concurrent Refinement Thread#21" os_prio=0 tid=0x00007f0ea805b800 nid=0xa185d runnable 

"G1 Concurrent Refinement Thread#22" os_prio=0 tid=0x00007f0ea805a000 nid=0xa185c runnable 

"G1 Concurrent Refinement Thread#23" os_prio=0 tid=0x00007f0ea8058000 nid=0xa185b runnable 

"VM Periodic Task Thread" os_prio=0 tid=0x00007f0ea8820800 nid=0xa1895 waiting on condition 

JNI global references: 4104