2019-05-02_15:17:16.30638 2019-05-02 15:17:16 2019-05-02_15:17:16.30777 Full thread dump Java HotSpot(TM) 64-Bit Server VM (25.102-b14 mixed mode): 2019-05-02_15:17:16.30777 2019-05-02_15:17:16.30777 "executor-Produce" #78 prio=5 os_prio=0 tid=0x00007f5fd8018800 nid=0x67f9 waiting on condition [0x00007f5f0825b000] 2019-05-02_15:17:16.30778 java.lang.Thread.State: WAITING (parking) 2019-05-02_15:17:16.30779 at sun.misc.Unsafe.park(Native Method) 2019-05-02_15:17:16.30779 - parking to wait for <0x0000000681e332b0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) 2019-05-02_15:17:16.30779 at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) 2019-05-02_15:17:16.30779 at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) 2019-05-02_15:17:16.30780 at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) 2019-05-02_15:17:16.30780 at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1067) 2019-05-02_15:17:16.30780 at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127) 2019-05-02_15:17:16.30780 at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) 2019-05-02_15:17:16.30780 at java.lang.Thread.run(Thread.java:745) 2019-05-02_15:17:16.30781 2019-05-02_15:17:16.30781 "ReplicaFetcherThread-0-1" #77 prio=5 os_prio=0 tid=0x00007f6038959800 nid=0x33b0 runnable [0x00007f5f2d3f8000] 2019-05-02_15:17:16.30781 java.lang.Thread.State: RUNNABLE 2019-05-02_15:17:16.30782 at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) 2019-05-02_15:17:16.30782 at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) 2019-05-02_15:17:16.30783 at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) 2019-05-02_15:17:16.30783 at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) 2019-05-02_15:17:16.30783 - locked <0x0000000685150e80> (a sun.nio.ch.Util$3) 2019-05-02_15:17:16.30783 - locked <0x0000000685150e90> (a java.util.Collections$UnmodifiableSet) 2019-05-02_15:17:16.30784 - locked <0x0000000685150e38> (a sun.nio.ch.EPollSelectorImpl) 2019-05-02_15:17:16.30784 at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) 2019-05-02_15:17:16.30784 at org.apache.kafka.common.network.Selector.select(Selector.java:752) 2019-05-02_15:17:16.30784 at org.apache.kafka.common.network.Selector.poll(Selector.java:451) 2019-05-02_15:17:16.30784 at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:521) 2019-05-02_15:17:16.30785 at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) 2019-05-02_15:17:16.30785 at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:97) 2019-05-02_15:17:16.30785 at kafka.server.ReplicaFetcherThread.fetchFromLeader(ReplicaFetcherThread.scala:190) 2019-05-02_15:17:16.30785 at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:241) 2019-05-02_15:17:16.30785 at kafka.server.AbstractFetcherThread.$anonfun$maybeFetch$3(AbstractFetcherThread.scala:130) 2019-05-02_15:17:16.30786 at kafka.server.AbstractFetcherThread.$anonfun$maybeFetch$3$adapted(AbstractFetcherThread.scala:129) 2019-05-02_15:17:16.30786 at kafka.server.AbstractFetcherThread$$Lambda$704/1592346659.apply(Unknown Source) 2019-05-02_15:17:16.30787 at scala.Option.foreach(Option.scala:257) 2019-05-02_15:17:16.30787 at kafka.server.AbstractFetcherThread.maybeFetch(AbstractFetcherThread.scala:129) 2019-05-02_15:17:16.30787 at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:111) 2019-05-02_15:17:16.30787 at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) 2019-05-02_15:17:16.30787 2019-05-02_15:17:16.30788 "ReplicaFetcherThread-0-2" #76 prio=5 os_prio=0 tid=0x00007f60393b0000 nid=0x3202 runnable [0x00007f5f2d0f5000] 2019-05-02_15:17:16.30788 java.lang.Thread.State: RUNNABLE 2019-05-02_15:17:16.30788 at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) 2019-05-02_15:17:16.30788 at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) 2019-05-02_15:17:16.30788 at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) 2019-05-02_15:17:16.30789 at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) 2019-05-02_15:17:16.30789 - locked <0x0000000685091298> (a sun.nio.ch.Util$3) 2019-05-02_15:17:16.30789 - locked <0x00000006850912a8> (a java.util.Collections$UnmodifiableSet) 2019-05-02_15:17:16.30790 - locked <0x0000000685091250> (a sun.nio.ch.EPollSelectorImpl) 2019-05-02_15:17:16.30790 at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) 2019-05-02_15:17:16.30790 at org.apache.kafka.common.network.Selector.select(Selector.java:752) 2019-05-02_15:17:16.30790 at org.apache.kafka.common.network.Selector.poll(Selector.java:451) 2019-05-02_15:17:16.30790 at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:521) 2019-05-02_15:17:16.30791 at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) 2019-05-02_15:17:16.30791 at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:97) 2019-05-02_15:17:16.30791 at kafka.server.ReplicaFetcherThread.fetchFromLeader(ReplicaFetcherThread.scala:190) 2019-05-02_15:17:16.30791 at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:241) 2019-05-02_15:17:16.30791 at kafka.server.AbstractFetcherThread.$anonfun$maybeFetch$3(AbstractFetcherThread.scala:130) 2019-05-02_15:17:16.30792 at kafka.server.AbstractFetcherThread.$anonfun$maybeFetch$3$adapted(AbstractFetcherThread.scala:129) 2019-05-02_15:17:16.30792 at kafka.server.AbstractFetcherThread$$Lambda$704/1592346659.apply(Unknown Source) 2019-05-02_15:17:16.30792 at scala.Option.foreach(Option.scala:257) 2019-05-02_15:17:16.30793 at kafka.server.AbstractFetcherThread.maybeFetch(AbstractFetcherThread.scala:129) 2019-05-02_15:17:16.30793 at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:111) 2019-05-02_15:17:16.30793 at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) 2019-05-02_15:17:16.30794 2019-05-02_15:17:16.30794 "executor-Heartbeat" #74 prio=5 os_prio=0 tid=0x00007f6000002000 nid=0x1dea waiting on condition [0x00007f5f9f350000] 2019-05-02_15:17:16.30794 java.lang.Thread.State: WAITING (parking) 2019-05-02_15:17:16.30794 at sun.misc.Unsafe.park(Native Method) 2019-05-02_15:17:16.30794 - parking to wait for <0x0000000681ba0590> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) 2019-05-02_15:17:16.30795 at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) 2019-05-02_15:17:16.30795 at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) 2019-05-02_15:17:16.30795 at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) 2019-05-02_15:17:16.30795 at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1067) 2019-05-02_15:17:16.30795 at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127) 2019-05-02_15:17:16.30796 at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) 2019-05-02_15:17:16.30796 at java.lang.Thread.run(Thread.java:745) 2019-05-02_15:17:16.30797 2019-05-02_15:17:16.30797 "executor-Rebalance" #72 prio=5 os_prio=0 tid=0x00007f600c001800 nid=0x7613 waiting on condition [0x00007f5f950e6000] 2019-05-02_15:17:16.30797 java.lang.Thread.State: WAITING (parking) 2019-05-02_15:17:16.30797 at sun.misc.Unsafe.park(Native Method) 2019-05-02_15:17:16.30797 - parking to wait for <0x0000000681e34dc0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) 2019-05-02_15:17:16.30798 at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) 2019-05-02_15:17:16.30798 at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) 2019-05-02_15:17:16.30798 at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) 2019-05-02_15:17:16.30798 at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1067) 2019-05-02_15:17:16.30798 at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127) 2019-05-02_15:17:16.30799 at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) 2019-05-02_15:17:16.30799 at java.lang.Thread.run(Thread.java:745) 2019-05-02_15:17:16.30799 2019-05-02_15:17:16.30799 "executor-Fetch" #71 prio=5 os_prio=0 tid=0x00007f5fe4009800 nid=0x5d56 waiting on condition [0x00007f5f9ed03000] 2019-05-02_15:17:16.30800 java.lang.Thread.State: WAITING (parking) 2019-05-02_15:17:16.30800 at sun.misc.Unsafe.park(Native Method) 2019-05-02_15:17:16.30800 - parking to wait for <0x00000006818d0f60> (a java.util.concurrent.locks.ReentrantReadWriteLock$NonfairSync) 2019-05-02_15:17:16.30800 at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) 2019-05-02_15:17:16.30801 at java.util.concurrent.locks.AbstractQueuedSynchronizer.parkAndCheckInterrupt(AbstractQueuedSynchronizer.java:836) 2019-05-02_15:17:16.30801 at java.util.concurrent.locks.AbstractQueuedSynchronizer.doAcquireShared(AbstractQueuedSynchronizer.java:967) 2019-05-02_15:17:16.30801 at java.util.concurrent.locks.AbstractQueuedSynchronizer.acquireShared(AbstractQueuedSynchronizer.java:1283) 2019-05-02_15:17:16.30801 at java.util.concurrent.locks.ReentrantReadWriteLock$ReadLock.lock(ReentrantReadWriteLock.java:727) 2019-05-02_15:17:16.30801 at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:249) 2019-05-02_15:17:16.30802 at kafka.utils.CoreUtils$.inReadLock(CoreUtils.scala:257) 2019-05-02_15:17:16.30802 at kafka.cluster.Partition.readRecords(Partition.scala:765) 2019-05-02_15:17:16.30802 at kafka.server.ReplicaManager.read$1(ReplicaManager.scala:904) 2019-05-02_15:17:16.30803 at kafka.server.ReplicaManager.$anonfun$readFromLocalLog$4(ReplicaManager.scala:965) 2019-05-02_15:17:16.30803 at kafka.server.ReplicaManager$$Lambda$929/1111102343.apply(Unknown Source) 2019-05-02_15:17:16.30803 at scala.collection.mutable.ResizableArray.foreach(ResizableArray.scala:58) 2019-05-02_15:17:16.30804 at scala.collection.mutable.ResizableArray.foreach$(ResizableArray.scala:51) 2019-05-02_15:17:16.30804 at scala.collection.mutable.ArrayBuffer.foreach(ArrayBuffer.scala:47) 2019-05-02_15:17:16.30804 at kafka.server.ReplicaManager.readFromLocalLog(ReplicaManager.scala:964) 2019-05-02_15:17:16.30804 at kafka.server.DelayedFetch.onComplete(DelayedFetch.scala:160) 2019-05-02_15:17:16.30804 at kafka.server.DelayedOperation.forceComplete(DelayedOperation.scala:70) 2019-05-02_15:17:16.30805 at kafka.server.DelayedOperation.run(DelayedOperation.scala:143) 2019-05-02_15:17:16.30805 at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:511) 2019-05-02_15:17:16.30805 at java.util.concurrent.FutureTask.run(FutureTask.java:266) 2019-05-02_15:17:16.30805 at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142) 2019-05-02_15:17:16.30805 at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) 2019-05-02_15:17:16.30806 at java.lang.Thread.run(Thread.java:745) 2019-05-02_15:17:16.30806 2019-05-02_15:17:16.30806 "ReplicaFetcherThread-0-4" #70 prio=5 os_prio=0 tid=0x00007f601c107000 nid=0x5be5 runnable [0x00007f5f2cff4000] 2019-05-02_15:17:16.30807 java.lang.Thread.State: RUNNABLE 2019-05-02_15:17:16.30807 at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) 2019-05-02_15:17:16.30807 at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) 2019-05-02_15:17:16.30807 at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) 2019-05-02_15:17:16.30807 at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) 2019-05-02_15:17:16.30808 - locked <0x000000068180a208> (a sun.nio.ch.Util$3) 2019-05-02_15:17:16.30808 - locked <0x000000068180a218> (a java.util.Collections$UnmodifiableSet) 2019-05-02_15:17:16.30808 - locked <0x000000068180a1c0> (a sun.nio.ch.EPollSelectorImpl) 2019-05-02_15:17:16.30808 at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) 2019-05-02_15:17:16.30808 at org.apache.kafka.common.network.Selector.select(Selector.java:752) 2019-05-02_15:17:16.30809 at org.apache.kafka.common.network.Selector.poll(Selector.java:451) 2019-05-02_15:17:16.30809 at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:521) 2019-05-02_15:17:16.30809 at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) 2019-05-02_15:17:16.30809 at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:97) 2019-05-02_15:17:16.30810 at kafka.server.ReplicaFetcherThread.fetchFromLeader(ReplicaFetcherThread.scala:190) 2019-05-02_15:17:16.30810 at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:241) 2019-05-02_15:17:16.30810 at kafka.server.AbstractFetcherThread.$anonfun$maybeFetch$3(AbstractFetcherThread.scala:130) 2019-05-02_15:17:16.30810 at kafka.server.AbstractFetcherThread.$anonfun$maybeFetch$3$adapted(AbstractFetcherThread.scala:129) 2019-05-02_15:17:16.30811 at kafka.server.AbstractFetcherThread$$Lambda$704/1592346659.apply(Unknown Source) 2019-05-02_15:17:16.30811 at scala.Option.foreach(Option.scala:257) 2019-05-02_15:17:16.30811 at kafka.server.AbstractFetcherThread.maybeFetch(AbstractFetcherThread.scala:129) 2019-05-02_15:17:16.30811 at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:111) 2019-05-02_15:17:16.30812 at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) 2019-05-02_15:17:16.30812 2019-05-02_15:17:16.30812 "ReplicaFetcherThread-0-0" #68 prio=5 os_prio=0 tid=0x00007f601c0d1800 nid=0x5be3 runnable [0x00007f5f2d1f6000] 2019-05-02_15:17:16.30812 java.lang.Thread.State: RUNNABLE 2019-05-02_15:17:16.30813 at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) 2019-05-02_15:17:16.30813 at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) 2019-05-02_15:17:16.30813 at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) 2019-05-02_15:17:16.30814 at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) 2019-05-02_15:17:16.30814 - locked <0x00000006818131c0> (a sun.nio.ch.Util$3) 2019-05-02_15:17:16.30814 - locked <0x00000006818131d0> (a java.util.Collections$UnmodifiableSet) 2019-05-02_15:17:16.30814 - locked <0x0000000681813178> (a sun.nio.ch.EPollSelectorImpl) 2019-05-02_15:17:16.30814 at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) 2019-05-02_15:17:16.30815 at org.apache.kafka.common.network.Selector.select(Selector.java:752) 2019-05-02_15:17:16.30815 at org.apache.kafka.common.network.Selector.poll(Selector.java:451) 2019-05-02_15:17:16.30815 at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:521) 2019-05-02_15:17:16.30815 at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) 2019-05-02_15:17:16.30815 at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:97) 2019-05-02_15:17:16.30816 at kafka.server.ReplicaFetcherThread.fetchFromLeader(ReplicaFetcherThread.scala:190) 2019-05-02_15:17:16.30816 at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:241) 2019-05-02_15:17:16.30816 at kafka.server.AbstractFetcherThread.$anonfun$maybeFetch$3(AbstractFetcherThread.scala:130) 2019-05-02_15:17:16.30817 at kafka.server.AbstractFetcherThread.$anonfun$maybeFetch$3$adapted(AbstractFetcherThread.scala:129) 2019-05-02_15:17:16.30817 at kafka.server.AbstractFetcherThread$$Lambda$704/1592346659.apply(Unknown Source) 2019-05-02_15:17:16.30817 at scala.Option.foreach(Option.scala:257) 2019-05-02_15:17:16.30817 at kafka.server.AbstractFetcherThread.maybeFetch(AbstractFetcherThread.scala:129) 2019-05-02_15:17:16.30818 at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:111) 2019-05-02_15:17:16.30818 at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) 2019-05-02_15:17:16.30818 2019-05-02_15:17:16.30818 "ReplicaFetcherThread-0-5" #67 prio=5 os_prio=0 tid=0x00007f601c0b7000 nid=0x5be2 runnable [0x00007f5f2d2f7000] 2019-05-02_15:17:16.30818 java.lang.Thread.State: RUNNABLE 2019-05-02_15:17:16.30819 at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) 2019-05-02_15:17:16.30819 at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) 2019-05-02_15:17:16.30819 at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) 2019-05-02_15:17:16.30819 at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) 2019-05-02_15:17:16.30819 - locked <0x000000068180cef8> (a sun.nio.ch.Util$3) 2019-05-02_15:17:16.30820 - locked <0x0000000681a46240> (a java.util.Collections$UnmodifiableSet) 2019-05-02_15:17:16.30820 - locked <0x0000000681a46048> (a sun.nio.ch.EPollSelectorImpl) 2019-05-02_15:17:16.30820 at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) 2019-05-02_15:17:16.30821 at org.apache.kafka.common.network.Selector.select(Selector.java:752) 2019-05-02_15:17:16.30821 at org.apache.kafka.common.network.Selector.poll(Selector.java:451) 2019-05-02_15:17:16.30821 at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:521) 2019-05-02_15:17:16.30821 at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) 2019-05-02_15:17:16.30821 at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:97) 2019-05-02_15:17:16.30822 at kafka.server.ReplicaFetcherThread.fetchFromLeader(ReplicaFetcherThread.scala:190) 2019-05-02_15:17:16.30822 at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:241) 2019-05-02_15:17:16.30822 at kafka.server.AbstractFetcherThread.$anonfun$maybeFetch$3(AbstractFetcherThread.scala:130) 2019-05-02_15:17:16.30822 at kafka.server.AbstractFetcherThread.$anonfun$maybeFetch$3$adapted(AbstractFetcherThread.scala:129) 2019-05-02_15:17:16.30823 at kafka.server.AbstractFetcherThread$$Lambda$704/1592346659.apply(Unknown Source) 2019-05-02_15:17:16.30823 at scala.Option.foreach(Option.scala:257) 2019-05-02_15:17:16.30823 at kafka.server.AbstractFetcherThread.maybeFetch(AbstractFetcherThread.scala:129) 2019-05-02_15:17:16.30824 at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:111) 2019-05-02_15:17:16.30824 at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) 2019-05-02_15:17:16.30824 2019-05-02_15:17:16.30824 "kafka-network-thread-3-ListenerName(SSL)-SSL-5" #65 prio=5 os_prio=0 tid=0x00007f6038990800 nid=0x5be0 runnable [0x00007f5f2d6f9000] 2019-05-02_15:17:16.30824 java.lang.Thread.State: RUNNABLE 2019-05-02_15:17:16.30825 at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) 2019-05-02_15:17:16.30825 at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) 2019-05-02_15:17:16.30825 at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) 2019-05-02_15:17:16.30825 at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) 2019-05-02_15:17:16.30825 - locked <0x0000000681813870> (a sun.nio.ch.Util$3) 2019-05-02_15:17:16.30826 - locked <0x0000000681813880> (a java.util.Collections$UnmodifiableSet) 2019-05-02_15:17:16.30826 - locked <0x0000000681813828> (a sun.nio.ch.EPollSelectorImpl) 2019-05-02_15:17:16.30826 at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) 2019-05-02_15:17:16.30826 at org.apache.kafka.common.network.Selector.select(Selector.java:752) 2019-05-02_15:17:16.30826 at org.apache.kafka.common.network.Selector.poll(Selector.java:451) 2019-05-02_15:17:16.30827 at kafka.network.Processor.poll(SocketServer.scala:689) 2019-05-02_15:17:16.30827 at kafka.network.Processor.run(SocketServer.scala:594) 2019-05-02_15:17:16.30828 at java.lang.Thread.run(Thread.java:745) 2019-05-02_15:17:16.30828 2019-05-02_15:17:16.30828 "kafka-network-thread-3-ListenerName(SSL)-SSL-4" #64 prio=5 os_prio=0 tid=0x00007f603898f000 nid=0x5bdf runnable [0x00007f5f2d7fa000] 2019-05-02_15:17:16.30828 java.lang.Thread.State: RUNNABLE 2019-05-02_15:17:16.30828 at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) 2019-05-02_15:17:16.30829 at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) 2019-05-02_15:17:16.30829 at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) 2019-05-02_15:17:16.30829 at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) 2019-05-02_15:17:16.30829 - locked <0x0000000681813b90> (a sun.nio.ch.Util$3) 2019-05-02_15:17:16.30829 - locked <0x0000000681813ba0> (a java.util.Collections$UnmodifiableSet) 2019-05-02_15:17:16.30830 - locked <0x0000000681813b48> (a sun.nio.ch.EPollSelectorImpl) 2019-05-02_15:17:16.30830 at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) 2019-05-02_15:17:16.30830 at org.apache.kafka.common.network.Selector.select(Selector.java:752) 2019-05-02_15:17:16.30830 at org.apache.kafka.common.network.Selector.poll(Selector.java:451) 2019-05-02_15:17:16.30830 at kafka.network.Processor.poll(SocketServer.scala:689) 2019-05-02_15:17:16.30831 at kafka.network.Processor.run(SocketServer.scala:594) 2019-05-02_15:17:16.30831 at java.lang.Thread.run(Thread.java:745) 2019-05-02_15:17:16.30831 2019-05-02_15:17:16.30832 "kafka-network-thread-3-ListenerName(SSL)-SSL-3" #63 prio=5 os_prio=0 tid=0x00007f603898d000 nid=0x5bde runnable [0x00007f5f2edea000] 2019-05-02_15:17:16.30832 java.lang.Thread.State: RUNNABLE 2019-05-02_15:17:16.30832 at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) 2019-05-02_15:17:16.30832 at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) 2019-05-02_15:17:16.30832 at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) 2019-05-02_15:17:16.30833 at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) 2019-05-02_15:17:16.30833 - locked <0x000000068180a870> (a sun.nio.ch.Util$3) 2019-05-02_15:17:16.30833 - locked <0x000000068180a880> (a java.util.Collections$UnmodifiableSet) 2019-05-02_15:17:16.30833 - locked <0x000000068180a828> (a sun.nio.ch.EPollSelectorImpl) 2019-05-02_15:17:16.30834 at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) 2019-05-02_15:17:16.30834 at org.apache.kafka.common.network.Selector.select(Selector.java:752) 2019-05-02_15:17:16.30834 at org.apache.kafka.common.network.Selector.poll(Selector.java:451) 2019-05-02_15:17:16.30834 at kafka.network.Processor.poll(SocketServer.scala:689) 2019-05-02_15:17:16.30834 at kafka.network.Processor.run(SocketServer.scala:594) 2019-05-02_15:17:16.30835 at java.lang.Thread.run(Thread.java:745) 2019-05-02_15:17:16.30835 2019-05-02_15:17:16.30835 "kafka-network-thread-3-ListenerName(PLAINTEXT)-PLAINTEXT-2" #62 prio=5 os_prio=0 tid=0x00007f603898b800 nid=0x5bdd waiting on condition [0x00007f5f2eeeb000] 2019-05-02_15:17:16.30836 java.lang.Thread.State: WAITING (parking) 2019-05-02_15:17:16.30836 at sun.misc.Unsafe.park(Native Method) 2019-05-02_15:17:16.30836 - parking to wait for <0x0000000681b81410> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) 2019-05-02_15:17:16.30836 at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) 2019-05-02_15:17:16.30837 at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) 2019-05-02_15:17:16.30837 at java.util.concurrent.ArrayBlockingQueue.put(ArrayBlockingQueue.java:353) 2019-05-02_15:17:16.30837 at kafka.network.RequestChannel.sendRequest(RequestChannel.scala:310) 2019-05-02_15:17:16.30837 at kafka.network.Processor.$anonfun$processCompletedReceives$1(SocketServer.scala:709) 2019-05-02_15:17:16.30837 at kafka.network.Processor.$anonfun$processCompletedReceives$1$adapted(SocketServer.scala:699) 2019-05-02_15:17:16.30838 at kafka.network.Processor$$Lambda$579/657792443.apply(Unknown Source) 2019-05-02_15:17:16.30838 at scala.collection.Iterator.foreach(Iterator.scala:937) 2019-05-02_15:17:16.30838 at scala.collection.Iterator.foreach$(Iterator.scala:937) 2019-05-02_15:17:16.30839 at scala.collection.AbstractIterator.foreach(Iterator.scala:1425) 2019-05-02_15:17:16.30839 at scala.collection.IterableLike.foreach(IterableLike.scala:70) 2019-05-02_15:17:16.30839 at scala.collection.IterableLike.foreach$(IterableLike.scala:69) 2019-05-02_15:17:16.30839 at scala.collection.AbstractIterable.foreach(Iterable.scala:54) 2019-05-02_15:17:16.30839 at kafka.network.Processor.processCompletedReceives(SocketServer.scala:699) 2019-05-02_15:17:16.30840 at kafka.network.Processor.run(SocketServer.scala:595) 2019-05-02_15:17:16.30840 at java.lang.Thread.run(Thread.java:745) 2019-05-02_15:17:16.30840 2019-05-02_15:17:16.30840 "kafka-network-thread-3-ListenerName(PLAINTEXT)-PLAINTEXT-1" #61 prio=5 os_prio=0 tid=0x00007f6038989800 nid=0x5bdc waiting on condition [0x00007f5f2efec000] 2019-05-02_15:17:16.30840 java.lang.Thread.State: WAITING (parking) 2019-05-02_15:17:16.30841 at sun.misc.Unsafe.park(Native Method) 2019-05-02_15:17:16.30841 - parking to wait for <0x0000000681b81410> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) 2019-05-02_15:17:16.30841 at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) 2019-05-02_15:17:16.30841 at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) 2019-05-02_15:17:16.30842 at java.util.concurrent.ArrayBlockingQueue.put(ArrayBlockingQueue.java:353) 2019-05-02_15:17:16.30842 at kafka.network.RequestChannel.sendRequest(RequestChannel.scala:310) 2019-05-02_15:17:16.30842 at kafka.network.Processor.$anonfun$processCompletedReceives$1(SocketServer.scala:709) 2019-05-02_15:17:16.30843 at kafka.network.Processor.$anonfun$processCompletedReceives$1$adapted(SocketServer.scala:699) 2019-05-02_15:17:16.30843 at kafka.network.Processor$$Lambda$579/657792443.apply(Unknown Source) 2019-05-02_15:17:16.30843 at scala.collection.Iterator.foreach(Iterator.scala:937) 2019-05-02_15:17:16.30843 at scala.collection.Iterator.foreach$(Iterator.scala:937) 2019-05-02_15:17:16.30844 at scala.collection.AbstractIterator.foreach(Iterator.scala:1425) 2019-05-02_15:17:16.30844 at scala.collection.IterableLike.foreach(IterableLike.scala:70) 2019-05-02_15:17:16.30844 at scala.collection.IterableLike.foreach$(IterableLike.scala:69) 2019-05-02_15:17:16.30844 at scala.collection.AbstractIterable.foreach(Iterable.scala:54) 2019-05-02_15:17:16.30844 at kafka.network.Processor.processCompletedReceives(SocketServer.scala:699) 2019-05-02_15:17:16.30845 at kafka.network.Processor.run(SocketServer.scala:595) 2019-05-02_15:17:16.30845 at java.lang.Thread.run(Thread.java:745) 2019-05-02_15:17:16.30845 2019-05-02_15:17:16.30846 "kafka-network-thread-3-ListenerName(PLAINTEXT)-PLAINTEXT-0" #60 prio=5 os_prio=0 tid=0x00007f6038988000 nid=0x5bdb waiting on condition [0x00007f5f2f0ed000] 2019-05-02_15:17:16.30846 java.lang.Thread.State: WAITING (parking) 2019-05-02_15:17:16.30846 at sun.misc.Unsafe.park(Native Method) 2019-05-02_15:17:16.30846 - parking to wait for <0x0000000681b81410> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) 2019-05-02_15:17:16.30847 at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) 2019-05-02_15:17:16.30847 at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) 2019-05-02_15:17:16.30847 at java.util.concurrent.ArrayBlockingQueue.put(ArrayBlockingQueue.java:353) 2019-05-02_15:17:16.30847 at kafka.network.RequestChannel.sendRequest(RequestChannel.scala:310) 2019-05-02_15:17:16.30848 at kafka.network.Processor.$anonfun$processCompletedReceives$1(SocketServer.scala:709) 2019-05-02_15:17:16.30848 at kafka.network.Processor.$anonfun$processCompletedReceives$1$adapted(SocketServer.scala:699) 2019-05-02_15:17:16.30848 at kafka.network.Processor$$Lambda$579/657792443.apply(Unknown Source) 2019-05-02_15:17:16.30849 at scala.collection.Iterator.foreach(Iterator.scala:937) 2019-05-02_15:17:16.30850 at scala.collection.Iterator.foreach$(Iterator.scala:937) 2019-05-02_15:17:16.30850 at scala.collection.AbstractIterator.foreach(Iterator.scala:1425) 2019-05-02_15:17:16.30850 at scala.collection.IterableLike.foreach(IterableLike.scala:70) 2019-05-02_15:17:16.30850 at scala.collection.IterableLike.foreach$(IterableLike.scala:69) 2019-05-02_15:17:16.30851 at scala.collection.AbstractIterable.foreach(Iterable.scala:54) 2019-05-02_15:17:16.30851 at kafka.network.Processor.processCompletedReceives(SocketServer.scala:699) 2019-05-02_15:17:16.30851 at kafka.network.Processor.run(SocketServer.scala:595) 2019-05-02_15:17:16.30851 at java.lang.Thread.run(Thread.java:745) 2019-05-02_15:17:16.30852 2019-05-02_15:17:16.30852 "/config/changes-event-process-thread" #59 prio=5 os_prio=0 tid=0x00007f6038982800 nid=0x5bda waiting on condition [0x00007f5f2f1ee000] 2019-05-02_15:17:16.30852 java.lang.Thread.State: WAITING (parking) 2019-05-02_15:17:16.30852 at sun.misc.Unsafe.park(Native Method) 2019-05-02_15:17:16.30852 - parking to wait for <0x000000068180b158> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) 2019-05-02_15:17:16.30853 at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) 2019-05-02_15:17:16.30853 at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) 2019-05-02_15:17:16.30853 at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) 2019-05-02_15:17:16.30854 at kafka.common.ZkNodeChangeNotificationListener$ChangeEventProcessThread.doWork(ZkNodeChangeNotificationListener.scala:145) 2019-05-02_15:17:16.30854 at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) 2019-05-02_15:17:16.30854 2019-05-02_15:17:16.30854 "kafka-request-handler-7" #58 daemon prio=5 os_prio=0 tid=0x00007f603896f800 nid=0x5bd9 waiting on condition [0x00007f5f2f2ef000] 2019-05-02_15:17:16.30855 java.lang.Thread.State: WAITING (parking) 2019-05-02_15:17:16.30855 at sun.misc.Unsafe.park(Native Method) 2019-05-02_15:17:16.30855 - parking to wait for <0x00000006818cc7e8> (a java.util.concurrent.locks.ReentrantReadWriteLock$NonfairSync) 2019-05-02_15:17:16.30856 at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) 2019-05-02_15:17:16.30856 at java.util.concurrent.locks.AbstractQueuedSynchronizer.parkAndCheckInterrupt(AbstractQueuedSynchronizer.java:836) 2019-05-02_15:17:16.30856 at java.util.concurrent.locks.AbstractQueuedSynchronizer.doAcquireShared(AbstractQueuedSynchronizer.java:967) 2019-05-02_15:17:16.30856 at java.util.concurrent.locks.AbstractQueuedSynchronizer.acquireShared(AbstractQueuedSynchronizer.java:1283) 2019-05-02_15:17:16.30856 at java.util.concurrent.locks.ReentrantReadWriteLock$ReadLock.lock(ReentrantReadWriteLock.java:727) 2019-05-02_15:17:16.30857 at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:249) 2019-05-02_15:17:16.30857 at kafka.utils.CoreUtils$.inReadLock(CoreUtils.scala:257) 2019-05-02_15:17:16.30858 at kafka.cluster.Partition.appendRecordsToLeader(Partition.scala:729) 2019-05-02_15:17:16.30858 at kafka.server.ReplicaManager.$anonfun$appendToLocalLog$2(ReplicaManager.scala:735) 2019-05-02_15:17:16.30858 at kafka.server.ReplicaManager$$Lambda$942/1463223413.apply(Unknown Source) 2019-05-02_15:17:16.30858 at scala.collection.TraversableLike.$anonfun$map$1(TraversableLike.scala:233) 2019-05-02_15:17:16.30858 at scala.collection.TraversableLike$$Lambda$8/1858609436.apply(Unknown Source) 2019-05-02_15:17:16.30859 at scala.collection.mutable.HashMap.$anonfun$foreach$1(HashMap.scala:145) 2019-05-02_15:17:16.30859 at scala.collection.mutable.HashMap$$Lambda$20/599491651.apply(Unknown Source) 2019-05-02_15:17:16.30859 at scala.collection.mutable.HashTable.foreachEntry(HashTable.scala:235) 2019-05-02_15:17:16.30859 at scala.collection.mutable.HashTable.foreachEntry$(HashTable.scala:228) 2019-05-02_15:17:16.30859 at scala.collection.mutable.HashMap.foreachEntry(HashMap.scala:40) 2019-05-02_15:17:16.30860 at scala.collection.mutable.HashMap.foreach(HashMap.scala:145) 2019-05-02_15:17:16.30860 at scala.collection.TraversableLike.map(TraversableLike.scala:233) 2019-05-02_15:17:16.30860 at scala.collection.TraversableLike.map$(TraversableLike.scala:226) 2019-05-02_15:17:16.30861 at scala.collection.AbstractTraversable.map(Traversable.scala:104) 2019-05-02_15:17:16.30861 at kafka.server.ReplicaManager.appendToLocalLog(ReplicaManager.scala:723) 2019-05-02_15:17:16.30861 at kafka.server.ReplicaManager.appendRecords(ReplicaManager.scala:470) 2019-05-02_15:17:16.30861 at kafka.server.KafkaApis.handleProduceRequest(KafkaApis.scala:482) 2019-05-02_15:17:16.30861 at kafka.server.KafkaApis.handle(KafkaApis.scala:106) 2019-05-02_15:17:16.30862 at kafka.server.KafkaRequestHandler.run(KafkaRequestHandler.scala:69) 2019-05-02_15:17:16.30862 at java.lang.Thread.run(Thread.java:745) 2019-05-02_15:17:16.30862 2019-05-02_15:17:16.30862 "kafka-request-handler-6" #57 daemon prio=5 os_prio=0 tid=0x00007f603896d800 nid=0x5bd8 waiting on condition [0x00007f5f2f3f0000] 2019-05-02_15:17:16.30862 java.lang.Thread.State: WAITING (parking) 2019-05-02_15:17:16.30863 at sun.misc.Unsafe.park(Native Method) 2019-05-02_15:17:16.30863 - parking to wait for <0x00000006818cc7e8> (a java.util.concurrent.locks.ReentrantReadWriteLock$NonfairSync) 2019-05-02_15:17:16.30863 at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) 2019-05-02_15:17:16.30863 at java.util.concurrent.locks.AbstractQueuedSynchronizer.parkAndCheckInterrupt(AbstractQueuedSynchronizer.java:836) 2019-05-02_15:17:16.30864 at java.util.concurrent.locks.AbstractQueuedSynchronizer.doAcquireShared(AbstractQueuedSynchronizer.java:967) 2019-05-02_15:17:16.30864 at java.util.concurrent.locks.AbstractQueuedSynchronizer.acquireShared(AbstractQueuedSynchronizer.java:1283) 2019-05-02_15:17:16.30864 at java.util.concurrent.locks.ReentrantReadWriteLock$ReadLock.lock(ReentrantReadWriteLock.java:727) 2019-05-02_15:17:16.30865 at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:249) 2019-05-02_15:17:16.30865 at kafka.utils.CoreUtils$.inReadLock(CoreUtils.scala:257) 2019-05-02_15:17:16.30865 at kafka.cluster.Partition.fetchOffsetSnapshot(Partition.scala:832) 2019-05-02_15:17:16.30865 at kafka.server.DelayedFetch.$anonfun$tryComplete$1(DelayedFetch.scala:87) 2019-05-02_15:17:16.30866 at kafka.server.DelayedFetch.$anonfun$tryComplete$1$adapted(DelayedFetch.scala:79) 2019-05-02_15:17:16.30866 at kafka.server.DelayedFetch$$Lambda$1036/1218231096.apply(Unknown Source) 2019-05-02_15:17:16.30866 at scala.collection.mutable.ResizableArray.foreach(ResizableArray.scala:58) 2019-05-02_15:17:16.30866 at scala.collection.mutable.ResizableArray.foreach$(ResizableArray.scala:51) 2019-05-02_15:17:16.30866 at scala.collection.mutable.ArrayBuffer.foreach(ArrayBuffer.scala:47) 2019-05-02_15:17:16.30867 at kafka.server.DelayedFetch.tryComplete(DelayedFetch.scala:79) 2019-05-02_15:17:16.30867 at kafka.server.DelayedOperation.maybeTryComplete(DelayedOperation.scala:121) 2019-05-02_15:17:16.30867 at kafka.server.DelayedOperationPurgatory$Watchers.tryCompleteWatched(DelayedOperation.scala:371) 2019-05-02_15:17:16.30868 at kafka.server.DelayedOperationPurgatory.checkAndComplete(DelayedOperation.scala:277) 2019-05-02_15:17:16.30868 at kafka.server.ReplicaManager.tryCompleteDelayedFetch(ReplicaManager.scala:307) 2019-05-02_15:17:16.30868 at kafka.cluster.Partition.$anonfun$appendRecordsToLeader$1(Partition.scala:743) 2019-05-02_15:17:16.30868 at kafka.cluster.Partition$$Lambda$961/1181816213.apply(Unknown Source) 2019-05-02_15:17:16.30869 at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) 2019-05-02_15:17:16.30869 at kafka.utils.CoreUtils$.inReadLock(CoreUtils.scala:257) 2019-05-02_15:17:16.30869 at kafka.cluster.Partition.appendRecordsToLeader(Partition.scala:729) 2019-05-02_15:17:16.30869 at kafka.server.ReplicaManager.$anonfun$appendToLocalLog$2(ReplicaManager.scala:735) 2019-05-02_15:17:16.30869 at kafka.server.ReplicaManager$$Lambda$942/1463223413.apply(Unknown Source) 2019-05-02_15:17:16.30870 at scala.collection.TraversableLike.$anonfun$map$1(TraversableLike.scala:233) 2019-05-02_15:17:16.30870 at scala.collection.TraversableLike$$Lambda$8/1858609436.apply(Unknown Source) 2019-05-02_15:17:16.30870 at scala.collection.mutable.HashMap.$anonfun$foreach$1(HashMap.scala:145) 2019-05-02_15:17:16.30871 at scala.collection.mutable.HashMap$$Lambda$20/599491651.apply(Unknown Source) 2019-05-02_15:17:16.30871 at scala.collection.mutable.HashTable.foreachEntry(HashTable.scala:235) 2019-05-02_15:17:16.30871 at scala.collection.mutable.HashTable.foreachEntry$(HashTable.scala:228) 2019-05-02_15:17:16.30871 at scala.collection.mutable.HashMap.foreachEntry(HashMap.scala:40) 2019-05-02_15:17:16.30871 at scala.collection.mutable.HashMap.foreach(HashMap.scala:145) 2019-05-02_15:17:16.30872 at scala.collection.TraversableLike.map(TraversableLike.scala:233) 2019-05-02_15:17:16.30872 at scala.collection.TraversableLike.map$(TraversableLike.scala:226) 2019-05-02_15:17:16.30872 at scala.collection.AbstractTraversable.map(Traversable.scala:104) 2019-05-02_15:17:16.30872 at kafka.server.ReplicaManager.appendToLocalLog(ReplicaManager.scala:723) 2019-05-02_15:17:16.30872 at kafka.server.ReplicaManager.appendRecords(ReplicaManager.scala:470) 2019-05-02_15:17:16.30873 at kafka.server.KafkaApis.handleProduceRequest(KafkaApis.scala:482) 2019-05-02_15:17:16.30873 at kafka.server.KafkaApis.handle(KafkaApis.scala:106) 2019-05-02_15:17:16.30873 at kafka.server.KafkaRequestHandler.run(KafkaRequestHandler.scala:69) 2019-05-02_15:17:16.30874 at java.lang.Thread.run(Thread.java:745) 2019-05-02_15:17:16.30874 2019-05-02_15:17:16.30874 "kafka-request-handler-5" #56 daemon prio=5 os_prio=0 tid=0x00007f603896c000 nid=0x5bd7 waiting on condition [0x00007f5f2f4f1000] 2019-05-02_15:17:16.30874 java.lang.Thread.State: WAITING (parking) 2019-05-02_15:17:16.30874 at sun.misc.Unsafe.park(Native Method) 2019-05-02_15:17:16.30875 - parking to wait for <0x00000006818d0f60> (a java.util.concurrent.locks.ReentrantReadWriteLock$NonfairSync) 2019-05-02_15:17:16.30875 at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) 2019-05-02_15:17:16.30875 at java.util.concurrent.locks.AbstractQueuedSynchronizer.parkAndCheckInterrupt(AbstractQueuedSynchronizer.java:836) 2019-05-02_15:17:16.30876 at java.util.concurrent.locks.AbstractQueuedSynchronizer.acquireQueued(AbstractQueuedSynchronizer.java:870) 2019-05-02_15:17:16.30876 at java.util.concurrent.locks.AbstractQueuedSynchronizer.acquire(AbstractQueuedSynchronizer.java:1199) 2019-05-02_15:17:16.30876 at java.util.concurrent.locks.ReentrantReadWriteLock$WriteLock.lock(ReentrantReadWriteLock.java:943) 2019-05-02_15:17:16.30876 at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:249) 2019-05-02_15:17:16.30876 at kafka.utils.CoreUtils$.inWriteLock(CoreUtils.scala:259) 2019-05-02_15:17:16.30877 at kafka.cluster.Partition.maybeExpandIsr(Partition.scala:503) 2019-05-02_15:17:16.30877 at kafka.cluster.Partition.updateReplicaLogReadResult(Partition.scala:473) 2019-05-02_15:17:16.30877 at kafka.server.ReplicaManager.$anonfun$updateFollowerLogReadResults$2(ReplicaManager.scala:1359) 2019-05-02_15:17:16.30878 at kafka.server.ReplicaManager$$Lambda$943/1005370159.apply(Unknown Source) 2019-05-02_15:17:16.30878 at scala.collection.TraversableLike.$anonfun$map$1(TraversableLike.scala:233) 2019-05-02_15:17:16.30878 at scala.collection.TraversableLike$$Lambda$8/1858609436.apply(Unknown Source) 2019-05-02_15:17:16.30878 at scala.collection.mutable.ResizableArray.foreach(ResizableArray.scala:58) 2019-05-02_15:17:16.30878 at scala.collection.mutable.ResizableArray.foreach$(ResizableArray.scala:51) 2019-05-02_15:17:16.30879 at scala.collection.mutable.ArrayBuffer.foreach(ArrayBuffer.scala:47) 2019-05-02_15:17:16.30879 at scala.collection.TraversableLike.map(TraversableLike.scala:233) 2019-05-02_15:17:16.30879 at scala.collection.TraversableLike.map$(TraversableLike.scala:226) 2019-05-02_15:17:16.30879 at scala.collection.AbstractTraversable.map(Traversable.scala:104) 2019-05-02_15:17:16.30879 at kafka.server.ReplicaManager.updateFollowerLogReadResults(ReplicaManager.scala:1353) 2019-05-02_15:17:16.30880 at kafka.server.ReplicaManager.readFromLog$1(ReplicaManager.scala:825) 2019-05-02_15:17:16.30880 at kafka.server.ReplicaManager.fetchMessages(ReplicaManager.scala:829) 2019-05-02_15:17:16.30881 at kafka.server.KafkaApis.handleFetchRequest(KafkaApis.scala:695) 2019-05-02_15:17:16.30881 at kafka.server.KafkaApis.handle(KafkaApis.scala:107) 2019-05-02_15:17:16.30881 at kafka.server.KafkaRequestHandler.run(KafkaRequestHandler.scala:69) 2019-05-02_15:17:16.30881 at java.lang.Thread.run(Thread.java:745) 2019-05-02_15:17:16.30881 2019-05-02_15:17:16.30882 "kafka-request-handler-4" #55 daemon prio=5 os_prio=0 tid=0x00007f603896a000 nid=0x5bd6 waiting on condition [0x00007f5f2f5f2000] 2019-05-02_15:17:16.30882 java.lang.Thread.State: WAITING (parking) 2019-05-02_15:17:16.30882 at sun.misc.Unsafe.park(Native Method) 2019-05-02_15:17:16.30882 - parking to wait for <0x00000006818cc7e8> (a java.util.concurrent.locks.ReentrantReadWriteLock$NonfairSync) 2019-05-02_15:17:16.30882 at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) 2019-05-02_15:17:16.30883 at java.util.concurrent.locks.AbstractQueuedSynchronizer.parkAndCheckInterrupt(AbstractQueuedSynchronizer.java:836) 2019-05-02_15:17:16.30883 at java.util.concurrent.locks.AbstractQueuedSynchronizer.acquireQueued(AbstractQueuedSynchronizer.java:870) 2019-05-02_15:17:16.30883 at java.util.concurrent.locks.AbstractQueuedSynchronizer.acquire(AbstractQueuedSynchronizer.java:1199) 2019-05-02_15:17:16.30884 at java.util.concurrent.locks.ReentrantReadWriteLock$WriteLock.lock(ReentrantReadWriteLock.java:943) 2019-05-02_15:17:16.30884 at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:249) 2019-05-02_15:17:16.30884 at kafka.utils.CoreUtils$.inWriteLock(CoreUtils.scala:259) 2019-05-02_15:17:16.30885 at kafka.cluster.Partition.maybeExpandIsr(Partition.scala:503) 2019-05-02_15:17:16.30885 at kafka.cluster.Partition.updateReplicaLogReadResult(Partition.scala:473) 2019-05-02_15:17:16.30885 at kafka.server.ReplicaManager.$anonfun$updateFollowerLogReadResults$2(ReplicaManager.scala:1359) 2019-05-02_15:17:16.30885 at kafka.server.ReplicaManager$$Lambda$943/1005370159.apply(Unknown Source) 2019-05-02_15:17:16.30885 at scala.collection.TraversableLike.$anonfun$map$1(TraversableLike.scala:233) 2019-05-02_15:17:16.30886 at scala.collection.TraversableLike$$Lambda$8/1858609436.apply(Unknown Source) 2019-05-02_15:17:16.30886 at scala.collection.mutable.ResizableArray.foreach(ResizableArray.scala:58) 2019-05-02_15:17:16.30886 at scala.collection.mutable.ResizableArray.foreach$(ResizableArray.scala:51) 2019-05-02_15:17:16.30886 at scala.collection.mutable.ArrayBuffer.foreach(ArrayBuffer.scala:47) 2019-05-02_15:17:16.30886 at scala.collection.TraversableLike.map(TraversableLike.scala:233) 2019-05-02_15:17:16.30887 at scala.collection.TraversableLike.map$(TraversableLike.scala:226) 2019-05-02_15:17:16.30887 at scala.collection.AbstractTraversable.map(Traversable.scala:104) 2019-05-02_15:17:16.30887 at kafka.server.ReplicaManager.updateFollowerLogReadResults(ReplicaManager.scala:1353) 2019-05-02_15:17:16.30888 at kafka.server.ReplicaManager.readFromLog$1(ReplicaManager.scala:825) 2019-05-02_15:17:16.30888 at kafka.server.ReplicaManager.fetchMessages(ReplicaManager.scala:829) 2019-05-02_15:17:16.30888 at kafka.server.KafkaApis.handleFetchRequest(KafkaApis.scala:695) 2019-05-02_15:17:16.30888 at kafka.server.KafkaApis.handle(KafkaApis.scala:107) 2019-05-02_15:17:16.30888 at kafka.server.KafkaRequestHandler.run(KafkaRequestHandler.scala:69) 2019-05-02_15:17:16.30889 at java.lang.Thread.run(Thread.java:745) 2019-05-02_15:17:16.30889 2019-05-02_15:17:16.30889 "kafka-request-handler-3" #54 daemon prio=5 os_prio=0 tid=0x00007f6038968800 nid=0x5bd5 waiting on condition [0x00007f5f2f6f3000] 2019-05-02_15:17:16.30889 java.lang.Thread.State: WAITING (parking) 2019-05-02_15:17:16.30889 at sun.misc.Unsafe.park(Native Method) 2019-05-02_15:17:16.30890 - parking to wait for <0x00000006818d0f60> (a java.util.concurrent.locks.ReentrantReadWriteLock$NonfairSync) 2019-05-02_15:17:16.30890 at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) 2019-05-02_15:17:16.30890 at java.util.concurrent.locks.AbstractQueuedSynchronizer.parkAndCheckInterrupt(AbstractQueuedSynchronizer.java:836) 2019-05-02_15:17:16.30891 at java.util.concurrent.locks.AbstractQueuedSynchronizer.doAcquireShared(AbstractQueuedSynchronizer.java:967) 2019-05-02_15:17:16.30891 at java.util.concurrent.locks.AbstractQueuedSynchronizer.acquireShared(AbstractQueuedSynchronizer.java:1283) 2019-05-02_15:17:16.30891 at java.util.concurrent.locks.ReentrantReadWriteLock$ReadLock.lock(ReentrantReadWriteLock.java:727) 2019-05-02_15:17:16.30891 at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:249) 2019-05-02_15:17:16.30892 at kafka.utils.CoreUtils$.inReadLock(CoreUtils.scala:257) 2019-05-02_15:17:16.30892 at kafka.cluster.Partition.readRecords(Partition.scala:765) 2019-05-02_15:17:16.30892 at kafka.server.ReplicaManager.read$1(ReplicaManager.scala:904) 2019-05-02_15:17:16.30892 at kafka.server.ReplicaManager.$anonfun$readFromLocalLog$4(ReplicaManager.scala:965) 2019-05-02_15:17:16.30892 at kafka.server.ReplicaManager$$Lambda$929/1111102343.apply(Unknown Source) 2019-05-02_15:17:16.30893 at scala.collection.mutable.ResizableArray.foreach(ResizableArray.scala:58) 2019-05-02_15:17:16.30893 at scala.collection.mutable.ResizableArray.foreach$(ResizableArray.scala:51) 2019-05-02_15:17:16.30893 at scala.collection.mutable.ArrayBuffer.foreach(ArrayBuffer.scala:47) 2019-05-02_15:17:16.30894 at kafka.server.ReplicaManager.readFromLocalLog(ReplicaManager.scala:964) 2019-05-02_15:17:16.30894 at kafka.server.ReplicaManager.readFromLog$1(ReplicaManager.scala:824) 2019-05-02_15:17:16.30894 at kafka.server.ReplicaManager.fetchMessages(ReplicaManager.scala:829) 2019-05-02_15:17:16.30894 at kafka.server.KafkaApis.handleFetchRequest(KafkaApis.scala:695) 2019-05-02_15:17:16.30895 at kafka.server.KafkaApis.handle(KafkaApis.scala:107) 2019-05-02_15:17:16.30895 at kafka.server.KafkaRequestHandler.run(KafkaRequestHandler.scala:69) 2019-05-02_15:17:16.30895 at java.lang.Thread.run(Thread.java:745) 2019-05-02_15:17:16.30895 2019-05-02_15:17:16.30895 "kafka-request-handler-2" #53 daemon prio=5 os_prio=0 tid=0x00007f6038967000 nid=0x5bd4 waiting on condition [0x00007f5f2f7f4000] 2019-05-02_15:17:16.30896 java.lang.Thread.State: WAITING (parking) 2019-05-02_15:17:16.30896 at sun.misc.Unsafe.park(Native Method) 2019-05-02_15:17:16.30896 - parking to wait for <0x00000006818d0f60> (a java.util.concurrent.locks.ReentrantReadWriteLock$NonfairSync) 2019-05-02_15:17:16.30896 at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) 2019-05-02_15:17:16.30896 at java.util.concurrent.locks.AbstractQueuedSynchronizer.parkAndCheckInterrupt(AbstractQueuedSynchronizer.java:836) 2019-05-02_15:17:16.30897 at java.util.concurrent.locks.AbstractQueuedSynchronizer.doAcquireShared(AbstractQueuedSynchronizer.java:967) 2019-05-02_15:17:16.30897 at java.util.concurrent.locks.AbstractQueuedSynchronizer.acquireShared(AbstractQueuedSynchronizer.java:1283) 2019-05-02_15:17:16.30898 at java.util.concurrent.locks.ReentrantReadWriteLock$ReadLock.lock(ReentrantReadWriteLock.java:727) 2019-05-02_15:17:16.30898 at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:249) 2019-05-02_15:17:16.30898 at kafka.utils.CoreUtils$.inReadLock(CoreUtils.scala:257) 2019-05-02_15:17:16.30898 at kafka.cluster.Partition.fetchOffsetSnapshot(Partition.scala:832) 2019-05-02_15:17:16.30898 at kafka.server.DelayedFetch.$anonfun$tryComplete$1(DelayedFetch.scala:87) 2019-05-02_15:17:16.30899 at kafka.server.DelayedFetch.$anonfun$tryComplete$1$adapted(DelayedFetch.scala:79) 2019-05-02_15:17:16.30899 at kafka.server.DelayedFetch$$Lambda$1036/1218231096.apply(Unknown Source) 2019-05-02_15:17:16.30899 at scala.collection.mutable.ResizableArray.foreach(ResizableArray.scala:58) 2019-05-02_15:17:16.30899 at scala.collection.mutable.ResizableArray.foreach$(ResizableArray.scala:51) 2019-05-02_15:17:16.30899 at scala.collection.mutable.ArrayBuffer.foreach(ArrayBuffer.scala:47) 2019-05-02_15:17:16.30900 at kafka.server.DelayedFetch.tryComplete(DelayedFetch.scala:79) 2019-05-02_15:17:16.30900 at kafka.server.DelayedOperation.maybeTryComplete(DelayedOperation.scala:121) 2019-05-02_15:17:16.30900 at kafka.server.DelayedOperationPurgatory$Watchers.tryCompleteWatched(DelayedOperation.scala:371) 2019-05-02_15:17:16.30903 at kafka.server.DelayedOperationPurgatory.checkAndComplete(DelayedOperation.scala:277) 2019-05-02_15:17:16.30903 at kafka.server.ReplicaManager.tryCompleteDelayedFetch(ReplicaManager.scala:307) 2019-05-02_15:17:16.30903 at kafka.cluster.Partition.$anonfun$appendRecordsToLeader$1(Partition.scala:743) 2019-05-02_15:17:16.30904 at kafka.cluster.Partition$$Lambda$961/1181816213.apply(Unknown Source) 2019-05-02_15:17:16.30904 at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) 2019-05-02_15:17:16.30904 at kafka.utils.CoreUtils$.inReadLock(CoreUtils.scala:257) 2019-05-02_15:17:16.30904 at kafka.cluster.Partition.appendRecordsToLeader(Partition.scala:729) 2019-05-02_15:17:16.30905 at kafka.server.ReplicaManager.$anonfun$appendToLocalLog$2(ReplicaManager.scala:735) 2019-05-02_15:17:16.30905 at kafka.server.ReplicaManager$$Lambda$942/1463223413.apply(Unknown Source) 2019-05-02_15:17:16.30905 at scala.collection.TraversableLike.$anonfun$map$1(TraversableLike.scala:233) 2019-05-02_15:17:16.30905 at scala.collection.TraversableLike$$Lambda$8/1858609436.apply(Unknown Source) 2019-05-02_15:17:16.30906 at scala.collection.mutable.HashMap.$anonfun$foreach$1(HashMap.scala:145) 2019-05-02_15:17:16.30906 at scala.collection.mutable.HashMap$$Lambda$20/599491651.apply(Unknown Source) 2019-05-02_15:17:16.30906 at scala.collection.mutable.HashTable.foreachEntry(HashTable.scala:235) 2019-05-02_15:17:16.30907 at scala.collection.mutable.HashTable.foreachEntry$(HashTable.scala:228) 2019-05-02_15:17:16.30907 at scala.collection.mutable.HashMap.foreachEntry(HashMap.scala:40) 2019-05-02_15:17:16.30907 at scala.collection.mutable.HashMap.foreach(HashMap.scala:145) 2019-05-02_15:17:16.30908 at scala.collection.TraversableLike.map(TraversableLike.scala:233) 2019-05-02_15:17:16.30908 at scala.collection.TraversableLike.map$(TraversableLike.scala:226) 2019-05-02_15:17:16.30908 at scala.collection.AbstractTraversable.map(Traversable.scala:104) 2019-05-02_15:17:16.30908 at kafka.server.ReplicaManager.appendToLocalLog(ReplicaManager.scala:723) 2019-05-02_15:17:16.30908 at kafka.server.ReplicaManager.appendRecords(ReplicaManager.scala:470) 2019-05-02_15:17:16.30909 at kafka.server.KafkaApis.handleProduceRequest(KafkaApis.scala:482) 2019-05-02_15:17:16.30909 at kafka.server.KafkaApis.handle(KafkaApis.scala:106) 2019-05-02_15:17:16.30909 at kafka.server.KafkaRequestHandler.run(KafkaRequestHandler.scala:69) 2019-05-02_15:17:16.30909 at java.lang.Thread.run(Thread.java:745) 2019-05-02_15:17:16.30909 2019-05-02_15:17:16.30910 "kafka-request-handler-1" #52 daemon prio=5 os_prio=0 tid=0x00007f6038964000 nid=0x5bd3 waiting on condition [0x00007f5f2f8f5000] 2019-05-02_15:17:16.30910 java.lang.Thread.State: WAITING (parking) 2019-05-02_15:17:16.30911 at sun.misc.Unsafe.park(Native Method) 2019-05-02_15:17:16.30911 - parking to wait for <0x00000006818cc7e8> (a java.util.concurrent.locks.ReentrantReadWriteLock$NonfairSync) 2019-05-02_15:17:16.30911 at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) 2019-05-02_15:17:16.30911 at java.util.concurrent.locks.AbstractQueuedSynchronizer.parkAndCheckInterrupt(AbstractQueuedSynchronizer.java:836) 2019-05-02_15:17:16.30911 at java.util.concurrent.locks.AbstractQueuedSynchronizer.doAcquireShared(AbstractQueuedSynchronizer.java:967) 2019-05-02_15:17:16.30912 at java.util.concurrent.locks.AbstractQueuedSynchronizer.acquireShared(AbstractQueuedSynchronizer.java:1283) 2019-05-02_15:17:16.30912 at java.util.concurrent.locks.ReentrantReadWriteLock$ReadLock.lock(ReentrantReadWriteLock.java:727) 2019-05-02_15:17:16.30912 at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:249) 2019-05-02_15:17:16.30912 at kafka.utils.CoreUtils$.inReadLock(CoreUtils.scala:257) 2019-05-02_15:17:16.30912 at kafka.cluster.Partition.appendRecordsToLeader(Partition.scala:729) 2019-05-02_15:17:16.30913 at kafka.server.ReplicaManager.$anonfun$appendToLocalLog$2(ReplicaManager.scala:735) 2019-05-02_15:17:16.30913 at kafka.server.ReplicaManager$$Lambda$942/1463223413.apply(Unknown Source) 2019-05-02_15:17:16.30913 at scala.collection.TraversableLike.$anonfun$map$1(TraversableLike.scala:233) 2019-05-02_15:17:16.30914 at scala.collection.TraversableLike$$Lambda$8/1858609436.apply(Unknown Source) 2019-05-02_15:17:16.30914 at scala.collection.mutable.HashMap.$anonfun$foreach$1(HashMap.scala:145) 2019-05-02_15:17:16.30914 at scala.collection.mutable.HashMap$$Lambda$20/599491651.apply(Unknown Source) 2019-05-02_15:17:16.30914 at scala.collection.mutable.HashTable.foreachEntry(HashTable.scala:235) 2019-05-02_15:17:16.30914 at scala.collection.mutable.HashTable.foreachEntry$(HashTable.scala:228) 2019-05-02_15:17:16.30915 at scala.collection.mutable.HashMap.foreachEntry(HashMap.scala:40) 2019-05-02_15:17:16.30915 at scala.collection.mutable.HashMap.foreach(HashMap.scala:145) 2019-05-02_15:17:16.30915 at scala.collection.TraversableLike.map(TraversableLike.scala:233) 2019-05-02_15:17:16.30915 at scala.collection.TraversableLike.map$(TraversableLike.scala:226) 2019-05-02_15:17:16.30916 at scala.collection.AbstractTraversable.map(Traversable.scala:104) 2019-05-02_15:17:16.30916 at kafka.server.ReplicaManager.appendToLocalLog(ReplicaManager.scala:723) 2019-05-02_15:17:16.30916 at kafka.server.ReplicaManager.appendRecords(ReplicaManager.scala:470) 2019-05-02_15:17:16.30917 at kafka.server.KafkaApis.handleProduceRequest(KafkaApis.scala:482) 2019-05-02_15:17:16.30917 at kafka.server.KafkaApis.handle(KafkaApis.scala:106) 2019-05-02_15:17:16.30917 at kafka.server.KafkaRequestHandler.run(KafkaRequestHandler.scala:69) 2019-05-02_15:17:16.30917 at java.lang.Thread.run(Thread.java:745) 2019-05-02_15:17:16.30918 2019-05-02_15:17:16.30918 "kafka-request-handler-0" #51 daemon prio=5 os_prio=0 tid=0x00007f6038962800 nid=0x5bd2 waiting on condition [0x00007f5f2f9f6000] 2019-05-02_15:17:16.30918 java.lang.Thread.State: WAITING (parking) 2019-05-02_15:17:16.30918 at sun.misc.Unsafe.park(Native Method) 2019-05-02_15:17:16.30918 - parking to wait for <0x00000006818cc7e8> (a java.util.concurrent.locks.ReentrantReadWriteLock$NonfairSync) 2019-05-02_15:17:16.30919 at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) 2019-05-02_15:17:16.30919 at java.util.concurrent.locks.AbstractQueuedSynchronizer.parkAndCheckInterrupt(AbstractQueuedSynchronizer.java:836) 2019-05-02_15:17:16.30919 at java.util.concurrent.locks.AbstractQueuedSynchronizer.doAcquireShared(AbstractQueuedSynchronizer.java:967) 2019-05-02_15:17:16.30919 at java.util.concurrent.locks.AbstractQueuedSynchronizer.acquireShared(AbstractQueuedSynchronizer.java:1283) 2019-05-02_15:17:16.30920 at java.util.concurrent.locks.ReentrantReadWriteLock$ReadLock.lock(ReentrantReadWriteLock.java:727) 2019-05-02_15:17:16.30920 at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:249) 2019-05-02_15:17:16.30920 at kafka.utils.CoreUtils$.inReadLock(CoreUtils.scala:257) 2019-05-02_15:17:16.30921 at kafka.cluster.Partition.appendRecordsToLeader(Partition.scala:729) 2019-05-02_15:17:16.30921 at kafka.server.ReplicaManager.$anonfun$appendToLocalLog$2(ReplicaManager.scala:735) 2019-05-02_15:17:16.30921 at kafka.server.ReplicaManager$$Lambda$942/1463223413.apply(Unknown Source) 2019-05-02_15:17:16.30921 at scala.collection.TraversableLike.$anonfun$map$1(TraversableLike.scala:233) 2019-05-02_15:17:16.30921 at scala.collection.TraversableLike$$Lambda$8/1858609436.apply(Unknown Source) 2019-05-02_15:17:16.30922 at scala.collection.mutable.HashMap.$anonfun$foreach$1(HashMap.scala:145) 2019-05-02_15:17:16.30922 at scala.collection.mutable.HashMap$$Lambda$20/599491651.apply(Unknown Source) 2019-05-02_15:17:16.30922 at scala.collection.mutable.HashTable.foreachEntry(HashTable.scala:235) 2019-05-02_15:17:16.30922 at scala.collection.mutable.HashTable.foreachEntry$(HashTable.scala:228) 2019-05-02_15:17:16.30922 at scala.collection.mutable.HashMap.foreachEntry(HashMap.scala:40) 2019-05-02_15:17:16.30923 at scala.collection.mutable.HashMap.foreach(HashMap.scala:145) 2019-05-02_15:17:16.30923 at scala.collection.TraversableLike.map(TraversableLike.scala:233) 2019-05-02_15:17:16.30923 at scala.collection.TraversableLike.map$(TraversableLike.scala:226) 2019-05-02_15:17:16.30924 at scala.collection.AbstractTraversable.map(Traversable.scala:104) 2019-05-02_15:17:16.30924 at kafka.server.ReplicaManager.appendToLocalLog(ReplicaManager.scala:723) 2019-05-02_15:17:16.30924 at kafka.server.ReplicaManager.appendRecords(ReplicaManager.scala:470) 2019-05-02_15:17:16.30924 at kafka.server.KafkaApis.handleProduceRequest(KafkaApis.scala:482) 2019-05-02_15:17:16.30924 at kafka.server.KafkaApis.handle(KafkaApis.scala:106) 2019-05-02_15:17:16.30925 at kafka.server.KafkaRequestHandler.run(KafkaRequestHandler.scala:69) 2019-05-02_15:17:16.30925 at java.lang.Thread.run(Thread.java:745) 2019-05-02_15:17:16.30925 2019-05-02_15:17:16.30925 "TxnMarkerSenderThread-3" #49 prio=5 os_prio=0 tid=0x00007f6038930800 nid=0x5bd1 runnable [0x00007f5f2faf7000] 2019-05-02_15:17:16.30925 java.lang.Thread.State: RUNNABLE 2019-05-02_15:17:16.30926 at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) 2019-05-02_15:17:16.30926 at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) 2019-05-02_15:17:16.30926 at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) 2019-05-02_15:17:16.30926 at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) 2019-05-02_15:17:16.30926 - locked <0x000000068180bb68> (a sun.nio.ch.Util$3) 2019-05-02_15:17:16.30927 - locked <0x000000068180bb78> (a java.util.Collections$UnmodifiableSet) 2019-05-02_15:17:16.30928 - locked <0x000000068180bb20> (a sun.nio.ch.EPollSelectorImpl) 2019-05-02_15:17:16.30928 at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) 2019-05-02_15:17:16.30928 at org.apache.kafka.common.network.Selector.select(Selector.java:752) 2019-05-02_15:17:16.30928 at org.apache.kafka.common.network.Selector.poll(Selector.java:451) 2019-05-02_15:17:16.30928 at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:521) 2019-05-02_15:17:16.30929 at kafka.common.InterBrokerSendThread.doWork(InterBrokerSendThread.scala:66) 2019-05-02_15:17:16.30929 at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) 2019-05-02_15:17:16.30929 2019-05-02_15:17:16.30929 "transaction-log-manager-0" #50 daemon prio=5 os_prio=0 tid=0x00007f603892e800 nid=0x5bd0 waiting on condition [0x00007f5f2fbf8000] 2019-05-02_15:17:16.30929 java.lang.Thread.State: TIMED_WAITING (parking) 2019-05-02_15:17:16.30930 at sun.misc.Unsafe.park(Native Method) 2019-05-02_15:17:16.30930 - parking to wait for <0x0000000681814820> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) 2019-05-02_15:17:16.30930 at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) 2019-05-02_15:17:16.30930 at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) 2019-05-02_15:17:16.30931 at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1093) 2019-05-02_15:17:16.30931 at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809) 2019-05-02_15:17:16.30931 at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1067) 2019-05-02_15:17:16.30931 at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127) 2019-05-02_15:17:16.30932 at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) 2019-05-02_15:17:16.30932 at java.lang.Thread.run(Thread.java:745) 2019-05-02_15:17:16.30932 2019-05-02_15:17:16.30932 "group-metadata-manager-0" #47 daemon prio=5 os_prio=0 tid=0x00007f60388bf000 nid=0x5bcf waiting on condition [0x00007f5f2fef9000] 2019-05-02_15:17:16.30932 java.lang.Thread.State: TIMED_WAITING (parking) 2019-05-02_15:17:16.30933 at sun.misc.Unsafe.park(Native Method) 2019-05-02_15:17:16.30933 - parking to wait for <0x000000068180bdd8> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) 2019-05-02_15:17:16.30933 at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) 2019-05-02_15:17:16.30934 at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) 2019-05-02_15:17:16.30934 at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1093) 2019-05-02_15:17:16.30934 at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809) 2019-05-02_15:17:16.30934 at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1067) 2019-05-02_15:17:16.30935 at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127) 2019-05-02_15:17:16.30935 at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) 2019-05-02_15:17:16.30935 at java.lang.Thread.run(Thread.java:745) 2019-05-02_15:17:16.30935 2019-05-02_15:17:16.30935 "ExpirationReaper-3-Rebalance" #46 prio=5 os_prio=0 tid=0x00007f6039611800 nid=0x5bce waiting on condition [0x00007f5f2fffa000] 2019-05-02_15:17:16.30936 java.lang.Thread.State: TIMED_WAITING (parking) 2019-05-02_15:17:16.30936 at sun.misc.Unsafe.park(Native Method) 2019-05-02_15:17:16.30936 - parking to wait for <0x0000000681814a80> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) 2019-05-02_15:17:16.30936 at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) 2019-05-02_15:17:16.30937 at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) 2019-05-02_15:17:16.30937 at java.util.concurrent.DelayQueue.poll(DelayQueue.java:259) 2019-05-02_15:17:16.30938 at kafka.utils.timer.SystemTimer.advanceClock(Timer.scala:105) 2019-05-02_15:17:16.30938 at kafka.server.DelayedOperationPurgatory.advanceClock(DelayedOperation.scala:416) 2019-05-02_15:17:16.30938 at kafka.server.DelayedOperationPurgatory$ExpiredOperationReaper.doWork(DelayedOperation.scala:440) 2019-05-02_15:17:16.30938 at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) 2019-05-02_15:17:16.30938 2019-05-02_15:17:16.30939 "ExpirationReaper-3-Heartbeat" #45 prio=5 os_prio=0 tid=0x00007f6039610000 nid=0x5bcd waiting on condition [0x00007f5f31595000] 2019-05-02_15:17:16.30939 java.lang.Thread.State: TIMED_WAITING (parking) 2019-05-02_15:17:16.30939 at sun.misc.Unsafe.park(Native Method) 2019-05-02_15:17:16.30939 - parking to wait for <0x000000068180c038> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) 2019-05-02_15:17:16.30939 at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) 2019-05-02_15:17:16.30940 at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) 2019-05-02_15:17:16.30940 at java.util.concurrent.DelayQueue.poll(DelayQueue.java:259) 2019-05-02_15:17:16.30940 at kafka.utils.timer.SystemTimer.advanceClock(Timer.scala:105) 2019-05-02_15:17:16.30941 at kafka.server.DelayedOperationPurgatory.advanceClock(DelayedOperation.scala:416) 2019-05-02_15:17:16.30941 at kafka.server.DelayedOperationPurgatory$ExpiredOperationReaper.doWork(DelayedOperation.scala:440) 2019-05-02_15:17:16.30941 at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) 2019-05-02_15:17:16.30941 2019-05-02_15:17:16.30941 "ExpirationReaper-3-topic" #44 prio=5 os_prio=0 tid=0x00007f6039609800 nid=0x5bcc waiting on condition [0x00007f5f31696000] 2019-05-02_15:17:16.30942 java.lang.Thread.State: TIMED_WAITING (parking) 2019-05-02_15:17:16.30942 at sun.misc.Unsafe.park(Native Method) 2019-05-02_15:17:16.30942 - parking to wait for <0x000000068180c288> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) 2019-05-02_15:17:16.30942 at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) 2019-05-02_15:17:16.30942 at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) 2019-05-02_15:17:16.30943 at java.util.concurrent.DelayQueue.poll(DelayQueue.java:259) 2019-05-02_15:17:16.30943 at kafka.utils.timer.SystemTimer.advanceClock(Timer.scala:105) 2019-05-02_15:17:16.30944 at kafka.server.DelayedOperationPurgatory.advanceClock(DelayedOperation.scala:416) 2019-05-02_15:17:16.30944 at kafka.server.DelayedOperationPurgatory$ExpiredOperationReaper.doWork(DelayedOperation.scala:440) 2019-05-02_15:17:16.30944 at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) 2019-05-02_15:17:16.30944 2019-05-02_15:17:16.30944 "controller-event-thread" #43 prio=5 os_prio=0 tid=0x00007f6039600000 nid=0x5bcb waiting on condition [0x00007f5f31797000] 2019-05-02_15:17:16.30945 java.lang.Thread.State: WAITING (parking) 2019-05-02_15:17:16.30945 at sun.misc.Unsafe.park(Native Method) 2019-05-02_15:17:16.30945 - parking to wait for <0x000000068180c4d8> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) 2019-05-02_15:17:16.30945 at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) 2019-05-02_15:17:16.30945 at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) 2019-05-02_15:17:16.30946 at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) 2019-05-02_15:17:16.30946 at kafka.controller.ControllerEventManager$ControllerEventThread.doWork(ControllerEventManager.scala:80) 2019-05-02_15:17:16.30946 at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) 2019-05-02_15:17:16.30947 2019-05-02_15:17:16.30947 "LogDirFailureHandler" #42 prio=5 os_prio=0 tid=0x00007f6039590000 nid=0x5bca waiting on condition [0x00007f5f31898000] 2019-05-02_15:17:16.30947 java.lang.Thread.State: WAITING (parking) 2019-05-02_15:17:16.30947 at sun.misc.Unsafe.park(Native Method) 2019-05-02_15:17:16.30948 - parking to wait for <0x00000006810026d8> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) 2019-05-02_15:17:16.30948 at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) 2019-05-02_15:17:16.30948 at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) 2019-05-02_15:17:16.30948 at java.util.concurrent.ArrayBlockingQueue.take(ArrayBlockingQueue.java:403) 2019-05-02_15:17:16.30948 at kafka.server.LogDirFailureChannel.takeNextOfflineLogDir(LogDirFailureChannel.scala:56) 2019-05-02_15:17:16.30949 at kafka.server.ReplicaManager$LogDirFailureHandler.doWork(ReplicaManager.scala:200) 2019-05-02_15:17:16.30949 at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) 2019-05-02_15:17:16.30949 2019-05-02_15:17:16.30949 "ExpirationReaper-3-DeleteRecords" #41 prio=5 os_prio=0 tid=0x00007f6039574800 nid=0x5bc9 waiting on condition [0x00007f5f31b99000] 2019-05-02_15:17:16.30950 java.lang.Thread.State: TIMED_WAITING (parking) 2019-05-02_15:17:16.30950 at sun.misc.Unsafe.park(Native Method) 2019-05-02_15:17:16.30950 - parking to wait for <0x0000000681814cd0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) 2019-05-02_15:17:16.30951 at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) 2019-05-02_15:17:16.30951 at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) 2019-05-02_15:17:16.30951 at java.util.concurrent.DelayQueue.poll(DelayQueue.java:259) 2019-05-02_15:17:16.30951 at kafka.utils.timer.SystemTimer.advanceClock(Timer.scala:105) 2019-05-02_15:17:16.30951 at kafka.server.DelayedOperationPurgatory.advanceClock(DelayedOperation.scala:416) 2019-05-02_15:17:16.30952 at kafka.server.DelayedOperationPurgatory$ExpiredOperationReaper.doWork(DelayedOperation.scala:440) 2019-05-02_15:17:16.30952 at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) 2019-05-02_15:17:16.30952 2019-05-02_15:17:16.30952 "ExpirationReaper-3-Fetch" #40 prio=5 os_prio=0 tid=0x00007f6039572800 nid=0x5bc8 waiting on condition [0x00007f5f31c9a000] 2019-05-02_15:17:16.30952 java.lang.Thread.State: TIMED_WAITING (parking) 2019-05-02_15:17:16.30953 at sun.misc.Unsafe.park(Native Method) 2019-05-02_15:17:16.30953 - parking to wait for <0x000000068180c8b0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) 2019-05-02_15:17:16.30953 at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) 2019-05-02_15:17:16.30954 at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) 2019-05-02_15:17:16.30954 at java.util.concurrent.DelayQueue.poll(DelayQueue.java:259) 2019-05-02_15:17:16.30954 at kafka.utils.timer.SystemTimer.advanceClock(Timer.scala:105) 2019-05-02_15:17:16.30954 at kafka.server.DelayedOperationPurgatory.advanceClock(DelayedOperation.scala:416) 2019-05-02_15:17:16.30954 at kafka.server.DelayedOperationPurgatory$ExpiredOperationReaper.doWork(DelayedOperation.scala:440) 2019-05-02_15:17:16.30955 at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) 2019-05-02_15:17:16.30955 2019-05-02_15:17:16.30955 "ExpirationReaper-3-Produce" #39 prio=5 os_prio=0 tid=0x00007f603956f800 nid=0x5bc7 waiting on condition [0x00007f5f31d9b000] 2019-05-02_15:17:16.30955 java.lang.Thread.State: TIMED_WAITING (parking) 2019-05-02_15:17:16.30955 at sun.misc.Unsafe.park(Native Method) 2019-05-02_15:17:16.30956 - parking to wait for <0x0000000681814f20> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) 2019-05-02_15:17:16.30956 at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) 2019-05-02_15:17:16.30957 at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) 2019-05-02_15:17:16.30957 at java.util.concurrent.DelayQueue.poll(DelayQueue.java:259) 2019-05-02_15:17:16.30957 at kafka.utils.timer.SystemTimer.advanceClock(Timer.scala:105) 2019-05-02_15:17:16.30957 at kafka.server.DelayedOperationPurgatory.advanceClock(DelayedOperation.scala:416) 2019-05-02_15:17:16.30958 at kafka.server.DelayedOperationPurgatory$ExpiredOperationReaper.doWork(DelayedOperation.scala:440) 2019-05-02_15:17:16.30958 at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) 2019-05-02_15:17:16.30958 2019-05-02_15:17:16.30958 "kafka-socket-acceptor-ListenerName(SSL)-SSL-9091" #38 prio=5 os_prio=0 tid=0x00007f603935e000 nid=0x5bc6 runnable [0x00007f5f31e9c000] 2019-05-02_15:17:16.30958 java.lang.Thread.State: RUNNABLE 2019-05-02_15:17:16.30959 at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) 2019-05-02_15:17:16.30959 at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) 2019-05-02_15:17:16.30959 at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) 2019-05-02_15:17:16.30959 at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) 2019-05-02_15:17:16.30960 - locked <0x000000068180cb78> (a sun.nio.ch.Util$3) 2019-05-02_15:17:16.30960 - locked <0x000000068180cb88> (a java.util.Collections$UnmodifiableSet) 2019-05-02_15:17:16.30960 - locked <0x000000068180cb30> (a sun.nio.ch.EPollSelectorImpl) 2019-05-02_15:17:16.30961 at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) 2019-05-02_15:17:16.30961 at kafka.network.Acceptor.run(SocketServer.scala:390) 2019-05-02_15:17:16.30961 at java.lang.Thread.run(Thread.java:745) 2019-05-02_15:17:16.30961 2019-05-02_15:17:16.30961 "kafka-socket-acceptor-ListenerName(PLAINTEXT)-PLAINTEXT-9092" #37 prio=5 os_prio=0 tid=0x00007f6039389800 nid=0x5bc5 runnable [0x00007f5f323fa000] 2019-05-02_15:17:16.30961 java.lang.Thread.State: RUNNABLE 2019-05-02_15:17:16.30962 at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) 2019-05-02_15:17:16.30962 at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) 2019-05-02_15:17:16.30962 at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) 2019-05-02_15:17:16.30962 at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) 2019-05-02_15:17:16.30962 - locked <0x00000006818151e8> (a sun.nio.ch.Util$3) 2019-05-02_15:17:16.30963 - locked <0x00000006818151f8> (a java.util.Collections$UnmodifiableSet) 2019-05-02_15:17:16.30963 - locked <0x00000006818151a0> (a sun.nio.ch.EPollSelectorImpl) 2019-05-02_15:17:16.30963 at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) 2019-05-02_15:17:16.30964 at kafka.network.Acceptor.run(SocketServer.scala:390) 2019-05-02_15:17:16.30964 at java.lang.Thread.run(Thread.java:745) 2019-05-02_15:17:16.30964 2019-05-02_15:17:16.30964 "kafka-log-cleaner-thread-0" #36 prio=5 os_prio=0 tid=0x00007f60384af000 nid=0x5bc3 waiting on condition [0x00007f5fec4cf000] 2019-05-02_15:17:16.30965 java.lang.Thread.State: TIMED_WAITING (parking) 2019-05-02_15:17:16.30965 at sun.misc.Unsafe.park(Native Method) 2019-05-02_15:17:16.30965 - parking to wait for <0x000000068180cd88> (a java.util.concurrent.CountDownLatch$Sync) 2019-05-02_15:17:16.30965 at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) 2019-05-02_15:17:16.30965 at java.util.concurrent.locks.AbstractQueuedSynchronizer.doAcquireSharedNanos(AbstractQueuedSynchronizer.java:1037) 2019-05-02_15:17:16.30966 at java.util.concurrent.locks.AbstractQueuedSynchronizer.tryAcquireSharedNanos(AbstractQueuedSynchronizer.java:1328) 2019-05-02_15:17:16.30966 at java.util.concurrent.CountDownLatch.await(CountDownLatch.java:277) 2019-05-02_15:17:16.30966 at kafka.utils.ShutdownableThread.pause(ShutdownableThread.scala:69) 2019-05-02_15:17:16.30966 at kafka.log.LogCleaner$CleanerThread.doWork(LogCleaner.scala:302) 2019-05-02_15:17:16.30967 at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) 2019-05-02_15:17:16.30967 2019-05-02_15:17:16.30967 "kafka-scheduler-9" #35 daemon prio=5 os_prio=0 tid=0x00007f5fa0025000 nid=0x5b31 waiting on condition [0x00007f5f9f451000] 2019-05-02_15:17:16.30968 java.lang.Thread.State: WAITING (parking) 2019-05-02_15:17:16.30968 at sun.misc.Unsafe.park(Native Method) 2019-05-02_15:17:16.30968 - parking to wait for <0x0000000681002750> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) 2019-05-02_15:17:16.30968 at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) 2019-05-02_15:17:16.30968 at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) 2019-05-02_15:17:16.30969 at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1088) 2019-05-02_15:17:16.30969 at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809) 2019-05-02_15:17:16.30969 at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1067) 2019-05-02_15:17:16.30969 at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127) 2019-05-02_15:17:16.30970 at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) 2019-05-02_15:17:16.30970 at java.lang.Thread.run(Thread.java:745) 2019-05-02_15:17:16.30970 2019-05-02_15:17:16.30970 "kafka-scheduler-8" #34 daemon prio=5 os_prio=0 tid=0x00007f5fa0163800 nid=0x5b30 waiting on condition [0x00007f5f9f552000] 2019-05-02_15:17:16.30971 java.lang.Thread.State: WAITING (parking) 2019-05-02_15:17:16.30971 at sun.misc.Unsafe.park(Native Method) 2019-05-02_15:17:16.30971 - parking to wait for <0x0000000681002750> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) 2019-05-02_15:17:16.30971 at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) 2019-05-02_15:17:16.30972 at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) 2019-05-02_15:17:16.30972 at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1088) 2019-05-02_15:17:16.30972 at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809) 2019-05-02_15:17:16.30972 at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1067) 2019-05-02_15:17:16.30972 at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127) 2019-05-02_15:17:16.30973 at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) 2019-05-02_15:17:16.30973 at java.lang.Thread.run(Thread.java:745) 2019-05-02_15:17:16.30973 2019-05-02_15:17:16.30974 "kafka-scheduler-7" #33 daemon prio=5 os_prio=0 tid=0x00007f5fa00a0000 nid=0x5b23 waiting on condition [0x00007f5f850fb000] 2019-05-02_15:17:16.30974 java.lang.Thread.State: WAITING (parking) 2019-05-02_15:17:16.30974 at sun.misc.Unsafe.park(Native Method) 2019-05-02_15:17:16.30974 - parking to wait for <0x0000000681002750> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) 2019-05-02_15:17:16.30974 at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) 2019-05-02_15:17:16.30975 at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) 2019-05-02_15:17:16.30975 at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1088) 2019-05-02_15:17:16.30975 at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809) 2019-05-02_15:17:16.30975 at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1067) 2019-05-02_15:17:16.30976 at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127) 2019-05-02_15:17:16.30976 at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) 2019-05-02_15:17:16.30976 at java.lang.Thread.run(Thread.java:745) 2019-05-02_15:17:16.30977 2019-05-02_15:17:16.30977 "kafka-scheduler-6" #32 daemon prio=5 os_prio=0 tid=0x00007f5fa009e000 nid=0x5b22 waiting on condition [0x00007f5f879fc000] 2019-05-02_15:17:16.30977 java.lang.Thread.State: WAITING (parking) 2019-05-02_15:17:16.30977 at sun.misc.Unsafe.park(Native Method) 2019-05-02_15:17:16.30978 - parking to wait for <0x00000006818cc7e8> (a java.util.concurrent.locks.ReentrantReadWriteLock$NonfairSync) 2019-05-02_15:17:16.30978 at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) 2019-05-02_15:17:16.30978 at java.util.concurrent.locks.AbstractQueuedSynchronizer.parkAndCheckInterrupt(AbstractQueuedSynchronizer.java:836) 2019-05-02_15:17:16.30978 at java.util.concurrent.locks.AbstractQueuedSynchronizer.acquireQueued(AbstractQueuedSynchronizer.java:870) 2019-05-02_15:17:16.30978 at java.util.concurrent.locks.AbstractQueuedSynchronizer.acquire(AbstractQueuedSynchronizer.java:1199) 2019-05-02_15:17:16.30979 at java.util.concurrent.locks.ReentrantReadWriteLock$WriteLock.lock(ReentrantReadWriteLock.java:943) 2019-05-02_15:17:16.30979 at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:249) 2019-05-02_15:17:16.30979 at kafka.utils.CoreUtils$.inWriteLock(CoreUtils.scala:259) 2019-05-02_15:17:16.30979 at kafka.cluster.Partition.maybeShrinkIsr(Partition.scala:633) 2019-05-02_15:17:16.30980 at kafka.server.ReplicaManager.$anonfun$maybeShrinkIsr$2(ReplicaManager.scala:1338) 2019-05-02_15:17:16.30980 at kafka.server.ReplicaManager.$anonfun$maybeShrinkIsr$2$adapted(ReplicaManager.scala:1338) 2019-05-02_15:17:16.30980 at kafka.server.ReplicaManager$$Lambda$456/2129458503.apply(Unknown Source) 2019-05-02_15:17:16.30981 at scala.collection.Iterator.foreach(Iterator.scala:937) 2019-05-02_15:17:16.30981 at scala.collection.Iterator.foreach$(Iterator.scala:937) 2019-05-02_15:17:16.30981 at scala.collection.AbstractIterator.foreach(Iterator.scala:1425) 2019-05-02_15:17:16.30981 at kafka.server.ReplicaManager.maybeShrinkIsr(ReplicaManager.scala:1338) 2019-05-02_15:17:16.30981 at kafka.server.ReplicaManager.$anonfun$startup$1(ReplicaManager.scala:323) 2019-05-02_15:17:16.30982 at kafka.server.ReplicaManager$$Lambda$449/452842611.apply$mcV$sp(Unknown Source) 2019-05-02_15:17:16.30982 at kafka.utils.KafkaScheduler.$anonfun$schedule$2(KafkaScheduler.scala:114) 2019-05-02_15:17:16.30982 at kafka.utils.KafkaScheduler$$Lambda$270/1525755354.apply$mcV$sp(Unknown Source) 2019-05-02_15:17:16.30982 at kafka.utils.CoreUtils$$anon$1.run(CoreUtils.scala:63) 2019-05-02_15:17:16.30982 at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:511) 2019-05-02_15:17:16.30983 at java.util.concurrent.FutureTask.runAndReset(FutureTask.java:308) 2019-05-02_15:17:16.30983 at java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.access$301(ScheduledThreadPoolExecutor.java:180) 2019-05-02_15:17:16.30983 at java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.run(ScheduledThreadPoolExecutor.java:294) 2019-05-02_15:17:16.30984 at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142) 2019-05-02_15:17:16.30984 at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) 2019-05-02_15:17:16.30984 at java.lang.Thread.run(Thread.java:745) 2019-05-02_15:17:16.30984 2019-05-02_15:17:16.30984 "kafka-scheduler-5" #31 daemon prio=5 os_prio=0 tid=0x00007f5fa009c000 nid=0x5b21 waiting on condition [0x00007f5f8b6fd000] 2019-05-02_15:17:16.30985 java.lang.Thread.State: WAITING (parking) 2019-05-02_15:17:16.30985 at sun.misc.Unsafe.park(Native Method) 2019-05-02_15:17:16.30985 - parking to wait for <0x0000000681002750> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) 2019-05-02_15:17:16.30985 at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) 2019-05-02_15:17:16.30985 at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) 2019-05-02_15:17:16.30986 at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1088) 2019-05-02_15:17:16.30987 at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809) 2019-05-02_15:17:16.30987 at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1067) 2019-05-02_15:17:16.30987 at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127) 2019-05-02_15:17:16.30987 at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) 2019-05-02_15:17:16.30988 at java.lang.Thread.run(Thread.java:745) 2019-05-02_15:17:16.30988 2019-05-02_15:17:16.30988 "kafka-scheduler-4" #30 daemon prio=5 os_prio=0 tid=0x00007f5fa009a800 nid=0x5b20 waiting on condition [0x00007f5f8dffe000] 2019-05-02_15:17:16.30988 java.lang.Thread.State: TIMED_WAITING (parking) 2019-05-02_15:17:16.30988 at sun.misc.Unsafe.park(Native Method) 2019-05-02_15:17:16.30989 - parking to wait for <0x0000000681002750> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) 2019-05-02_15:17:16.30989 at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) 2019-05-02_15:17:16.30989 at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) 2019-05-02_15:17:16.30989 at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1093) 2019-05-02_15:17:16.30990 at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809) 2019-05-02_15:17:16.30990 at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1067) 2019-05-02_15:17:16.30990 at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127) 2019-05-02_15:17:16.30991 at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) 2019-05-02_15:17:16.30991 at java.lang.Thread.run(Thread.java:745) 2019-05-02_15:17:16.30991 2019-05-02_15:17:16.30991 "kafka-scheduler-3" #29 daemon prio=5 os_prio=0 tid=0x00007f5fa0099000 nid=0x5b1f waiting on condition [0x00007f5f9c1fc000] 2019-05-02_15:17:16.30991 java.lang.Thread.State: WAITING (parking) 2019-05-02_15:17:16.30992 at sun.misc.Unsafe.park(Native Method) 2019-05-02_15:17:16.30992 - parking to wait for <0x0000000681002750> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) 2019-05-02_15:17:16.30992 at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) 2019-05-02_15:17:16.30992 at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) 2019-05-02_15:17:16.30992 at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1088) 2019-05-02_15:17:16.30993 at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809) 2019-05-02_15:17:16.30993 at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1067) 2019-05-02_15:17:16.30993 at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127) 2019-05-02_15:17:16.30994 at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) 2019-05-02_15:17:16.30994 at java.lang.Thread.run(Thread.java:745) 2019-05-02_15:17:16.30994 2019-05-02_15:17:16.30994 "kafka-scheduler-2" #28 daemon prio=5 os_prio=0 tid=0x00007f5fa0097800 nid=0x5b1e waiting on condition [0x00007f5f9c2fd000] 2019-05-02_15:17:16.30994 java.lang.Thread.State: WAITING (parking) 2019-05-02_15:17:16.30995 at sun.misc.Unsafe.park(Native Method) 2019-05-02_15:17:16.30995 - parking to wait for <0x0000000681002750> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) 2019-05-02_15:17:16.30995 at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) 2019-05-02_15:17:16.30995 at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) 2019-05-02_15:17:16.30996 at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1088) 2019-05-02_15:17:16.30996 at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809) 2019-05-02_15:17:16.30997 at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1067) 2019-05-02_15:17:16.30997 at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127) 2019-05-02_15:17:16.30997 at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) 2019-05-02_15:17:16.30997 at java.lang.Thread.run(Thread.java:745) 2019-05-02_15:17:16.30997 2019-05-02_15:17:16.30998 "kafka-scheduler-1" #27 daemon prio=5 os_prio=0 tid=0x00007f5fa007d800 nid=0x5b1d waiting on condition [0x00007f5f9c3fe000] 2019-05-02_15:17:16.30998 java.lang.Thread.State: WAITING (parking) 2019-05-02_15:17:16.30998 at sun.misc.Unsafe.park(Native Method) 2019-05-02_15:17:16.30998 - parking to wait for <0x0000000681002750> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) 2019-05-02_15:17:16.30998 at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) 2019-05-02_15:17:16.30999 at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) 2019-05-02_15:17:16.30999 at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1088) 2019-05-02_15:17:16.31000 at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809) 2019-05-02_15:17:16.31000 at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1067) 2019-05-02_15:17:16.31000 at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127) 2019-05-02_15:17:16.31000 at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) 2019-05-02_15:17:16.31000 at java.lang.Thread.run(Thread.java:745) 2019-05-02_15:17:16.31001 2019-05-02_15:17:16.31001 "kafka-scheduler-0" #26 daemon prio=5 os_prio=0 tid=0x00007f5fa007c000 nid=0x5b1c waiting on condition [0x00007f5fec1ce000] 2019-05-02_15:17:16.31001 java.lang.Thread.State: WAITING (parking) 2019-05-02_15:17:16.31001 at sun.misc.Unsafe.park(Native Method) 2019-05-02_15:17:16.31001 - parking to wait for <0x0000000681002750> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) 2019-05-02_15:17:16.31002 at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) 2019-05-02_15:17:16.31002 at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) 2019-05-02_15:17:16.31002 at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1088) 2019-05-02_15:17:16.31003 at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809) 2019-05-02_15:17:16.31003 at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1067) 2019-05-02_15:17:16.31003 at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127) 2019-05-02_15:17:16.31003 at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) 2019-05-02_15:17:16.31004 at java.lang.Thread.run(Thread.java:745) 2019-05-02_15:17:16.31004 2019-05-02_15:17:16.31004 "ThrottledChannelReaper-Request" #24 prio=5 os_prio=0 tid=0x00007f603a655800 nid=0x5b17 waiting on condition [0x00007f5fec5d0000] 2019-05-02_15:17:16.31004 java.lang.Thread.State: TIMED_WAITING (parking) 2019-05-02_15:17:16.31004 at sun.misc.Unsafe.park(Native Method) 2019-05-02_15:17:16.31005 - parking to wait for <0x0000000681003720> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) 2019-05-02_15:17:16.31005 at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) 2019-05-02_15:17:16.31005 at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) 2019-05-02_15:17:16.31006 at java.util.concurrent.DelayQueue.poll(DelayQueue.java:259) 2019-05-02_15:17:16.31006 at kafka.server.ClientQuotaManager$ThrottledChannelReaper.doWork(ClientQuotaManager.scala:196) 2019-05-02_15:17:16.31006 at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) 2019-05-02_15:17:16.31006 2019-05-02_15:17:16.31007 "ThrottledChannelReaper-Produce" #23 prio=5 os_prio=0 tid=0x00007f603a650800 nid=0x5b16 waiting on condition [0x00007f5fec6d1000] 2019-05-02_15:17:16.31007 java.lang.Thread.State: TIMED_WAITING (parking) 2019-05-02_15:17:16.31007 at sun.misc.Unsafe.park(Native Method) 2019-05-02_15:17:16.31007 - parking to wait for <0x0000000681003900> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) 2019-05-02_15:17:16.31007 at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) 2019-05-02_15:17:16.31008 at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) 2019-05-02_15:17:16.31008 at java.util.concurrent.DelayQueue.poll(DelayQueue.java:259) 2019-05-02_15:17:16.31008 at kafka.server.ClientQuotaManager$ThrottledChannelReaper.doWork(ClientQuotaManager.scala:196) 2019-05-02_15:17:16.31008 at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) 2019-05-02_15:17:16.31009 2019-05-02_15:17:16.31009 "ThrottledChannelReaper-Fetch" #22 prio=5 os_prio=0 tid=0x00007f603a64f000 nid=0x5b15 waiting on condition [0x00007f5fec7d2000] 2019-05-02_15:17:16.31009 java.lang.Thread.State: TIMED_WAITING (parking) 2019-05-02_15:17:16.31010 at sun.misc.Unsafe.park(Native Method) 2019-05-02_15:17:16.31010 - parking to wait for <0x0000000681003ae0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) 2019-05-02_15:17:16.31010 at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) 2019-05-02_15:17:16.31010 at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) 2019-05-02_15:17:16.31010 at java.util.concurrent.DelayQueue.poll(DelayQueue.java:259) 2019-05-02_15:17:16.31011 at kafka.server.ClientQuotaManager$ThrottledChannelReaper.doWork(ClientQuotaManager.scala:196) 2019-05-02_15:17:16.31011 at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) 2019-05-02_15:17:16.31011 2019-05-02_15:17:16.31011 "SensorExpiryThread" #21 daemon prio=5 os_prio=0 tid=0x00007f603a63a800 nid=0x5b12 waiting on condition [0x00007f5fec8d3000] 2019-05-02_15:17:16.31011 java.lang.Thread.State: TIMED_WAITING (parking) 2019-05-02_15:17:16.31012 at sun.misc.Unsafe.park(Native Method) 2019-05-02_15:17:16.31012 - parking to wait for <0x0000000681003b28> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) 2019-05-02_15:17:16.31012 at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) 2019-05-02_15:17:16.31013 at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) 2019-05-02_15:17:16.31013 at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1093) 2019-05-02_15:17:16.31013 at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809) 2019-05-02_15:17:16.31013 at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1067) 2019-05-02_15:17:16.31014 at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127) 2019-05-02_15:17:16.31014 at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) 2019-05-02_15:17:16.31014 at java.lang.Thread.run(Thread.java:745) 2019-05-02_15:17:16.31014 2019-05-02_15:17:16.31014 "main-EventThread" #20 daemon prio=5 os_prio=0 tid=0x00007f603a4c1800 nid=0x5b11 waiting on condition [0x00007f5fecdd4000] 2019-05-02_15:17:16.31015 java.lang.Thread.State: WAITING (parking) 2019-05-02_15:17:16.31015 at sun.misc.Unsafe.park(Native Method) 2019-05-02_15:17:16.31016 - parking to wait for <0x0000000681003f48> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) 2019-05-02_15:17:16.31016 at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) 2019-05-02_15:17:16.31016 at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) 2019-05-02_15:17:16.31016 at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) 2019-05-02_15:17:16.31016 at org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:504) 2019-05-02_15:17:16.31017 2019-05-02_15:17:16.31017 "main-SendThread(zk2-events-2.ts.local:2181)" #19 daemon prio=5 os_prio=0 tid=0x00007f603a4c0000 nid=0x5b10 runnable [0x00007f5feced5000] 2019-05-02_15:17:16.31017 java.lang.Thread.State: RUNNABLE 2019-05-02_15:17:16.31017 at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) 2019-05-02_15:17:16.31017 at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) 2019-05-02_15:17:16.31018 at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) 2019-05-02_15:17:16.31018 at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) 2019-05-02_15:17:16.31018 - locked <0x00000006810041a8> (a sun.nio.ch.Util$3) 2019-05-02_15:17:16.31018 - locked <0x0000000681004198> (a java.util.Collections$UnmodifiableSet) 2019-05-02_15:17:16.31019 - locked <0x00000006810041b8> (a sun.nio.ch.EPollSelectorImpl) 2019-05-02_15:17:16.31019 at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) 2019-05-02_15:17:16.31019 at org.apache.zookeeper.ClientCnxnSocketNIO.doTransport(ClientCnxnSocketNIO.java:349) 2019-05-02_15:17:16.31019 at org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1145) 2019-05-02_15:17:16.31020 2019-05-02_15:17:16.31020 "metrics-meter-tick-thread-2" #18 daemon prio=5 os_prio=0 tid=0x00007f603a48f000 nid=0x5b0f waiting on condition [0x00007f5fed1d6000] 2019-05-02_15:17:16.31020 java.lang.Thread.State: WAITING (parking) 2019-05-02_15:17:16.31020 at sun.misc.Unsafe.park(Native Method) 2019-05-02_15:17:16.31020 - parking to wait for <0x0000000681004488> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) 2019-05-02_15:17:16.31021 at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) 2019-05-02_15:17:16.31021 at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) 2019-05-02_15:17:16.31021 at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1088) 2019-05-02_15:17:16.31021 at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809) 2019-05-02_15:17:16.31022 at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1067) 2019-05-02_15:17:16.31022 at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127) 2019-05-02_15:17:16.31022 at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) 2019-05-02_15:17:16.31023 at java.lang.Thread.run(Thread.java:745) 2019-05-02_15:17:16.31023 2019-05-02_15:17:16.31023 "metrics-meter-tick-thread-1" #17 daemon prio=5 os_prio=0 tid=0x00007f603a48d000 nid=0x5b0e runnable [0x00007f5fed2d7000] 2019-05-02_15:17:16.31023 java.lang.Thread.State: TIMED_WAITING (parking) 2019-05-02_15:17:16.31023 at sun.misc.Unsafe.park(Native Method) 2019-05-02_15:17:16.31024 - parking to wait for <0x0000000681004488> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) 2019-05-02_15:17:16.31024 at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) 2019-05-02_15:17:16.31024 at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) 2019-05-02_15:17:16.31024 at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1093) 2019-05-02_15:17:16.31025 at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809) 2019-05-02_15:17:16.31025 at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1067) 2019-05-02_15:17:16.31026 at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127) 2019-05-02_15:17:16.31026 at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) 2019-05-02_15:17:16.31026 at java.lang.Thread.run(Thread.java:745) 2019-05-02_15:17:16.31026 2019-05-02_15:17:16.31026 "metrics-graphite-reporter-thread-1" #15 daemon prio=5 os_prio=0 tid=0x00007f603a423800 nid=0x5b0d waiting on condition [0x00007f5fed3d8000] 2019-05-02_15:17:16.31027 java.lang.Thread.State: TIMED_WAITING (parking) 2019-05-02_15:17:16.31027 at sun.misc.Unsafe.park(Native Method) 2019-05-02_15:17:16.31027 - parking to wait for <0x0000000681004898> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) 2019-05-02_15:17:16.31027 at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) 2019-05-02_15:17:16.31027 at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) 2019-05-02_15:17:16.31028 at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1093) 2019-05-02_15:17:16.31028 at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809) 2019-05-02_15:17:16.31029 at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1067) 2019-05-02_15:17:16.31029 at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127) 2019-05-02_15:17:16.31029 at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) 2019-05-02_15:17:16.31029 at java.lang.Thread.run(Thread.java:745) 2019-05-02_15:17:16.31029 2019-05-02_15:17:16.31030 "RMI TCP Accept-0" #13 daemon prio=5 os_prio=0 tid=0x00007f603884f000 nid=0x5b0b runnable [0x00007f5fedbda000] 2019-05-02_15:17:16.31030 java.lang.Thread.State: RUNNABLE 2019-05-02_15:17:16.31030 at java.net.PlainSocketImpl.socketAccept(Native Method) 2019-05-02_15:17:16.31030 at java.net.AbstractPlainSocketImpl.accept(AbstractPlainSocketImpl.java:409) 2019-05-02_15:17:16.31030 at java.net.ServerSocket.implAccept(ServerSocket.java:545) 2019-05-02_15:17:16.31031 at java.net.ServerSocket.accept(ServerSocket.java:513) 2019-05-02_15:17:16.31031 at sun.management.jmxremote.LocalRMIServerSocketFactory$1.accept(LocalRMIServerSocketFactory.java:52) 2019-05-02_15:17:16.31031 at sun.rmi.transport.tcp.TCPTransport$AcceptLoop.executeAcceptLoop(TCPTransport.java:400) 2019-05-02_15:17:16.31031 at sun.rmi.transport.tcp.TCPTransport$AcceptLoop.run(TCPTransport.java:372) 2019-05-02_15:17:16.31032 at java.lang.Thread.run(Thread.java:745) 2019-05-02_15:17:16.31032 2019-05-02_15:17:16.31032 "RMI TCP Accept-9999" #12 daemon prio=5 os_prio=0 tid=0x00007f6038847800 nid=0x5b0a runnable [0x00007f5fedcdb000] 2019-05-02_15:17:16.31032 java.lang.Thread.State: RUNNABLE 2019-05-02_15:17:16.31033 at java.net.PlainSocketImpl.socketAccept(Native Method) 2019-05-02_15:17:16.31033 at java.net.AbstractPlainSocketImpl.accept(AbstractPlainSocketImpl.java:409) 2019-05-02_15:17:16.31033 at java.net.ServerSocket.implAccept(ServerSocket.java:545) 2019-05-02_15:17:16.31033 at java.net.ServerSocket.accept(ServerSocket.java:513) 2019-05-02_15:17:16.31033 at sun.rmi.transport.tcp.TCPTransport$AcceptLoop.executeAcceptLoop(TCPTransport.java:400) 2019-05-02_15:17:16.31034 at sun.rmi.transport.tcp.TCPTransport$AcceptLoop.run(TCPTransport.java:372) 2019-05-02_15:17:16.31034 at java.lang.Thread.run(Thread.java:745) 2019-05-02_15:17:16.31034 2019-05-02_15:17:16.31034 "RMI TCP Accept-0" #11 daemon prio=5 os_prio=0 tid=0x00007f6038833000 nid=0x5b09 runnable [0x00007f5fedddc000] 2019-05-02_15:17:16.31034 java.lang.Thread.State: RUNNABLE 2019-05-02_15:17:16.31035 at java.net.PlainSocketImpl.socketAccept(Native Method) 2019-05-02_15:17:16.31035 at java.net.AbstractPlainSocketImpl.accept(AbstractPlainSocketImpl.java:409) 2019-05-02_15:17:16.31035 at java.net.ServerSocket.implAccept(ServerSocket.java:545) 2019-05-02_15:17:16.31035 at java.net.ServerSocket.accept(ServerSocket.java:513) 2019-05-02_15:17:16.31036 at sun.rmi.transport.tcp.TCPTransport$AcceptLoop.executeAcceptLoop(TCPTransport.java:400) 2019-05-02_15:17:16.31036 at sun.rmi.transport.tcp.TCPTransport$AcceptLoop.run(TCPTransport.java:372) 2019-05-02_15:17:16.31036 at java.lang.Thread.run(Thread.java:745) 2019-05-02_15:17:16.31037 2019-05-02_15:17:16.31037 "Service Thread" #9 daemon prio=9 os_prio=0 tid=0x00007f6038440000 nid=0x5b08 runnable [0x0000000000000000] 2019-05-02_15:17:16.31037 java.lang.Thread.State: RUNNABLE 2019-05-02_15:17:16.31037 2019-05-02_15:17:16.31037 "C1 CompilerThread2" #8 daemon prio=9 os_prio=0 tid=0x00007f6038433000 nid=0x5b07 waiting on condition [0x0000000000000000] 2019-05-02_15:17:16.31038 java.lang.Thread.State: RUNNABLE 2019-05-02_15:17:16.31038 2019-05-02_15:17:16.31038 "C2 CompilerThread1" #7 daemon prio=9 os_prio=0 tid=0x00007f6038431000 nid=0x5b06 waiting on condition [0x0000000000000000] 2019-05-02_15:17:16.31038 java.lang.Thread.State: RUNNABLE 2019-05-02_15:17:16.31038 2019-05-02_15:17:16.31039 "C2 CompilerThread0" #6 daemon prio=9 os_prio=0 tid=0x00007f603842e800 nid=0x5b05 waiting on condition [0x0000000000000000] 2019-05-02_15:17:16.31039 java.lang.Thread.State: RUNNABLE 2019-05-02_15:17:16.31039 2019-05-02_15:17:16.31039 "Signal Dispatcher" #5 daemon prio=9 os_prio=0 tid=0x00007f603842c800 nid=0x5b04 waiting on condition [0x0000000000000000] 2019-05-02_15:17:16.31039 java.lang.Thread.State: RUNNABLE 2019-05-02_15:17:16.31040 2019-05-02_15:17:16.31040 "Surrogate Locker Thread (Concurrent GC)" #4 daemon prio=9 os_prio=0 tid=0x00007f603842b000 nid=0x5b03 waiting on condition [0x0000000000000000] 2019-05-02_15:17:16.31040 java.lang.Thread.State: RUNNABLE 2019-05-02_15:17:16.31041 2019-05-02_15:17:16.31041 "Finalizer" #3 daemon prio=8 os_prio=0 tid=0x00007f60383f9800 nid=0x5b02 in Object.wait() [0x00007f5ff51fc000] 2019-05-02_15:17:16.31041 java.lang.Thread.State: WAITING (on object monitor) 2019-05-02_15:17:16.31041 at java.lang.Object.wait(Native Method) 2019-05-02_15:17:16.31041 at java.lang.ref.ReferenceQueue.remove(ReferenceQueue.java:143) 2019-05-02_15:17:16.31042 - locked <0x00000006810237e8> (a java.lang.ref.ReferenceQueue$Lock) 2019-05-02_15:17:16.31042 at java.lang.ref.ReferenceQueue.remove(ReferenceQueue.java:164) 2019-05-02_15:17:16.31042 at java.lang.ref.Finalizer$FinalizerThread.run(Finalizer.java:209) 2019-05-02_15:17:16.31042 2019-05-02_15:17:16.31042 "Reference Handler" #2 daemon prio=10 os_prio=0 tid=0x00007f60383f5000 nid=0x5b01 in Object.wait() [0x00007f5ff52fd000] 2019-05-02_15:17:16.31043 java.lang.Thread.State: WAITING (on object monitor) 2019-05-02_15:17:16.31043 at java.lang.Object.wait(Native Method) 2019-05-02_15:17:16.31043 at java.lang.Object.wait(Object.java:502) 2019-05-02_15:17:16.31043 at java.lang.ref.Reference.tryHandlePending(Reference.java:191) 2019-05-02_15:17:16.31043 - locked <0x0000000681023828> (a java.lang.ref.Reference$Lock) 2019-05-02_15:17:16.31044 at java.lang.ref.Reference$ReferenceHandler.run(Reference.java:153) 2019-05-02_15:17:16.31044 2019-05-02_15:17:16.31044 "main" #1 prio=5 os_prio=0 tid=0x00007f603800b800 nid=0x5af4 waiting on condition [0x00007f603e162000] 2019-05-02_15:17:16.31045 java.lang.Thread.State: WAITING (parking) 2019-05-02_15:17:16.31045 at sun.misc.Unsafe.park(Native Method) 2019-05-02_15:17:16.31045 - parking to wait for <0x000000068180cdf8> (a java.util.concurrent.CountDownLatch$Sync) 2019-05-02_15:17:16.31045 at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) 2019-05-02_15:17:16.31045 at java.util.concurrent.locks.AbstractQueuedSynchronizer.parkAndCheckInterrupt(AbstractQueuedSynchronizer.java:836) 2019-05-02_15:17:16.31046 at java.util.concurrent.locks.AbstractQueuedSynchronizer.doAcquireSharedInterruptibly(AbstractQueuedSynchronizer.java:997) 2019-05-02_15:17:16.31046 at java.util.concurrent.locks.AbstractQueuedSynchronizer.acquireSharedInterruptibly(AbstractQueuedSynchronizer.java:1304) 2019-05-02_15:17:16.31046 at java.util.concurrent.CountDownLatch.await(CountDownLatch.java:231) 2019-05-02_15:17:16.31047 at kafka.server.KafkaServer.awaitShutdown(KafkaServer.scala:644) 2019-05-02_15:17:16.31047 at kafka.server.KafkaServerStartable.awaitShutdown(KafkaServerStartable.scala:65) 2019-05-02_15:17:16.31047 at kafka.Kafka$.main(Kafka.scala:76) 2019-05-02_15:17:16.31048 at kafka.Kafka.main(Kafka.scala) 2019-05-02_15:17:16.31048 2019-05-02_15:17:16.31048 "VM Thread" os_prio=0 tid=0x00007f60383ed000 nid=0x5b00 runnable 2019-05-02_15:17:16.31048 2019-05-02_15:17:16.31048 "Gang worker#0 (Parallel GC Threads)" os_prio=0 tid=0x00007f6038022800 nid=0x5af5 runnable 2019-05-02_15:17:16.31049 2019-05-02_15:17:16.31049 "Gang worker#1 (Parallel GC Threads)" os_prio=0 tid=0x00007f6038024800 nid=0x5af6 runnable 2019-05-02_15:17:16.31049 2019-05-02_15:17:16.31049 "Gang worker#2 (Parallel GC Threads)" os_prio=0 tid=0x00007f6038026000 nid=0x5af7 runnable 2019-05-02_15:17:16.31049 2019-05-02_15:17:16.31050 "Gang worker#3 (Parallel GC Threads)" os_prio=0 tid=0x00007f6038028000 nid=0x5af8 runnable 2019-05-02_15:17:16.31050 2019-05-02_15:17:16.31050 "G1 Main Concurrent Mark GC Thread" os_prio=0 tid=0x00007f6038038800 nid=0x5afe runnable 2019-05-02_15:17:16.31050 2019-05-02_15:17:16.31050 "Gang worker#0 (G1 Parallel Marking Threads)" os_prio=0 tid=0x00007f603803a800 nid=0x5aff runnable 2019-05-02_15:17:16.31051 2019-05-02_15:17:16.31284 "G1 Concurrent Refinement Thread#0" os_prio=0 tid=0x00007f6038031800 nid=0x5afd runnable 2019-05-02_15:17:16.31284 2019-05-02_15:17:16.31284 "G1 Concurrent Refinement Thread#1" os_prio=0 tid=0x00007f6038030000 nid=0x5afc runnable 2019-05-02_15:17:16.31285 2019-05-02_15:17:16.31285 "G1 Concurrent Refinement Thread#2" os_prio=0 tid=0x00007f603802e000 nid=0x5afb runnable 2019-05-02_15:17:16.31285 2019-05-02_15:17:16.31286 "G1 Concurrent Refinement Thread#3" os_prio=0 tid=0x00007f603802c800 nid=0x5afa runnable 2019-05-02_15:17:16.31286 2019-05-02_15:17:16.31286 "G1 Concurrent Refinement Thread#4" os_prio=0 tid=0x00007f603802a800 nid=0x5af9 runnable 2019-05-02_15:17:16.31286 2019-05-02_15:17:16.31287 "VM Periodic Task Thread" os_prio=0 tid=0x00007f6038857000 nid=0x5b0c waiting on condition 2019-05-02_15:17:16.31287 2019-05-02_15:17:16.31327 JNI global references: 3303 2019-05-02_15:17:16.31327 2019-05-02_15:17:16.31392 Heap 2019-05-02_15:17:16.31392 garbage-first heap total 5373952K, used 1981182K [0x0000000678000000, 0x0000000679000a40, 0x00000007c0000000) 2019-05-02_15:17:16.31392 region size 16384K, 66 young (1081344K), 2 survivors (32768K) 2019-05-02_15:17:16.31393 Metaspace used 41345K, capacity 45890K, committed 46080K, reserved 1089536K 2019-05-02_15:17:16.31393 class space used 5307K, capacity 5792K, committed 5888K, reserved 1048576K 2019-05-02_15:17:16.31393