2022-05-16 23:49:34 Full thread dump OpenJDK 64-Bit Server VM (11.0.14+9 mixed mode): Threads class SMR info: _java_thread_list=0x00007fd0d0032190, length=207, elements={ 0x00007fd1b82a7000, 0x00007fd1b82ab000, 0x00007fd1b82c0000, 0x00007fd1b82c2000, 0x00007fd1b82c4800, 0x00007fd1b82c6800, 0x00007fd1b82c8800, 0x00007fd1b8374000, 0x00007fd124141000, 0x00007fd12465c800, 0x00007fd1248c5000, 0x00007fd124904000, 0x00007fd124906000, 0x00007fd124907800, 0x00007fd124909800, 0x00007fd12490b000, 0x00007fd12490d000, 0x00007fd12490f000, 0x00007fd124910800, 0x00007fd124922000, 0x00007fd124923000, 0x00007fd124971800, 0x00007fd1249b3800, 0x00007fd124a96000, 0x00007fd124a9a800, 0x00007fd124bb5000, 0x00007fd124bbc000, 0x00007fd124bbd000, 0x00007fd124bbf000, 0x00007fd124bc0800, 0x00007fd124bc2800, 0x00007fd124bda000, 0x00007fd124bdb000, 0x00007fd124bdc800, 0x00007fd124bde800, 0x00007fd124be0800, 0x00007fd124be2800, 0x00007fd124be4000, 0x00007fd124be6800, 0x00007fd124be8800, 0x00007fd124beb000, 0x00007fd124bed000, 0x00007fd124bee800, 0x00007fd124bf0800, 0x00007fd124c4d000, 0x00007fd124c4f800, 0x00007fd124c51000, 0x00007fd124c56800, 0x00007fd124c58000, 0x00007fd124c5a000, 0x00007fd124c35000, 0x00007fd124d1b000, 0x00007fd124d17800, 0x00007fd124d29800, 0x00007fd124d2b000, 0x00007fd124d4d000, 0x00007fd124d50000, 0x00007fd124d51800, 0x00007fd124d54800, 0x00007fd124d56000, 0x00007fd124d58000, 0x00007fd124d5a000, 0x00007fd124d5b800, 0x00007fd124d5d800, 0x00007fd124d5f000, 0x00007fd124d65800, 0x00007fd124d67000, 0x00007fd124d68800, 0x00007fd124d6a800, 0x00007fd124d6c800, 0x00007fd124d6e800, 0x00007fd0f004f000, 0x00007fd124c60800, 0x00007fd124d7a800, 0x00007fd124d78800, 0x00007fd124d90800, 0x00007fd124d92800, 0x00007fd124d98000, 0x00007fd124d9a800, 0x00007fd124d9b800, 0x00007fd124d9d000, 0x00007fd124d9f000, 0x00007fd124da0800, 0x00007fd124da2800, 0x00007fd124da4800, 0x00007fd124da6000, 0x00007fd124da8000, 0x00007fd0f0096800, 0x00007fd1607b2000, 0x00007fd0f0097800, 0x00007fd1607b3000, 0x00007fd1607b7800, 0x00007fd0f00a5000, 0x00007fd0b405b800, 0x00007fd0c800f000, 0x00007fd0c0040800, 0x00007fd0bc003000, 0x00007fd0f8002000, 0x00007fd1607a0000, 0x00007fd0f0062000, 0x00007fd124ed3000, 0x00007fd124fac800, 0x00007fd124fad800, 0x00007fd124fdd000, 0x00007fd12502a000, 0x00007fd125032000, 0x00007fd125035000, 0x00007fd12503a800, 0x00007fd1280a3000, 0x00007fd125068800, 0x00007fd125364800, 0x00007fd124818000, 0x00007fd12500a000, 0x00007fd12500b800, 0x00007fd134004800, 0x00007fd134005800, 0x00007fd184015000, 0x00007fd134007000, 0x00007fd134008800, 0x00007fd13400a800, 0x00007fd13400e800, 0x00007fd134010000, 0x00007fd134012000, 0x00007fd134014000, 0x00007fd134015800, 0x00007fd134017800, 0x00007fd134019000, 0x00007fd13401b000, 0x00007fd13401c800, 0x00007fd13401e800, 0x00007fd124846000, 0x00007fd12484e000, 0x00007fd12484f800, 0x00007fd124851800, 0x00007fd124853000, 0x00007fd124855000, 0x00007fd124856800, 0x00007fd124858800, 0x00007fd12485a000, 0x00007fd12485c000, 0x00007fd124604800, 0x00007fd124631800, 0x00007fd12470f000, 0x00007fd18001d000, 0x00007fd180023800, 0x00007fd180025000, 0x00007fd18002f000, 0x00007fd18007d800, 0x00007fd180134800, 0x00007fd180137000, 0x00007fd180146800, 0x00007fd180106800, 0x00007fd180199000, 0x00007fd1801e6000, 0x00007fd18020f800, 0x00007fd180219800, 0x00007fd180255800, 0x00007fd0fc01c000, 0x00007fd17c006800, 0x00007fd17c009000, 0x00007fd1802e4000, 0x00007fd1802e5800, 0x00007fd1802e7800, 0x00007fd1802e9800, 0x00007fd1802eb000, 0x00007fd1802ed000, 0x00007fd1802ee800, 0x00007fd1802f9800, 0x00007fd1802fd800, 0x00007fd180306800, 0x00007fd18030a000, 0x00007fd18030c800, 0x00007fd18800b800, 0x00007fd14005c800, 0x00007fd14c05f800, 0x00007fd0e400b000, 0x00007fd0e400d000, 0x00007fd180361000, 0x00007fd180361800, 0x00007fd124f12000, 0x00007fd120015000, 0x00007fd1247ad000, 0x00007fd13c006000, 0x00007fd0d0023000, 0x00007fd190024800, 0x00007fd1480e2800, 0x00007fd148056000, 0x00007fd148057000, 0x00007fd190026000, 0x00007fd190027000, 0x00007fd0ac035800, 0x00007fd190034000, 0x00007fd190035800, 0x00007fd14806d800, 0x00007fd1b9163000, 0x00007fd1b8027800, 0x00007fd12c18e000, 0x00007fd12462f800, 0x00007fd14806e800, 0x00007fd0f00d1000, 0x00007fd16801b800, 0x00007fd0f800a800, 0x00007fd16803b000, 0x00007fd0fc03e800, 0x00007fd108006000, 0x00007fd190036000, 0x00007fd170010000 } "Reference Handler" #2 daemon prio=10 os_prio=0 cpu=98.05ms elapsed=6579.51s tid=0x00007fd1b82a7000 nid=0x8aa5 waiting on condition [0x00007fd1945e6000] java.lang.Thread.State: RUNNABLE at java.lang.ref.Reference.waitForReferencePendingList(java.base@11.0.14/Native Method) at java.lang.ref.Reference.processPendingReferences(java.base@11.0.14/Reference.java:241) at java.lang.ref.Reference$ReferenceHandler.run(java.base@11.0.14/Reference.java:213) "Finalizer" #3 daemon prio=8 os_prio=0 cpu=75.25ms elapsed=6579.51s tid=0x00007fd1b82ab000 nid=0x8aa6 in Object.wait() [0x00007fd1944e6000] java.lang.Thread.State: WAITING (on object monitor) at java.lang.Object.wait(java.base@11.0.14/Native Method) - waiting on at java.lang.ref.ReferenceQueue.remove(java.base@11.0.14/ReferenceQueue.java:155) - waiting to re-lock in wait() <0x000000077708d268> (a java.lang.ref.ReferenceQueue$Lock) at java.lang.ref.ReferenceQueue.remove(java.base@11.0.14/ReferenceQueue.java:176) at java.lang.ref.Finalizer$FinalizerThread.run(java.base@11.0.14/Finalizer.java:170) "Signal Dispatcher" #4 daemon prio=9 os_prio=0 cpu=0.38ms elapsed=6579.50s tid=0x00007fd1b82c0000 nid=0x8aa7 runnable [0x0000000000000000] java.lang.Thread.State: RUNNABLE "Service Thread" #5 daemon prio=9 os_prio=0 cpu=0.12ms elapsed=6579.50s tid=0x00007fd1b82c2000 nid=0x8aa8 runnable [0x0000000000000000] java.lang.Thread.State: RUNNABLE "C2 CompilerThread0" #6 daemon prio=9 os_prio=0 cpu=30830.94ms elapsed=6579.50s tid=0x00007fd1b82c4800 nid=0x8aa9 waiting on condition [0x0000000000000000] java.lang.Thread.State: RUNNABLE No compile task "C1 CompilerThread0" #9 daemon prio=9 os_prio=0 cpu=5729.90ms elapsed=6579.50s tid=0x00007fd1b82c6800 nid=0x8aaa waiting on condition [0x0000000000000000] java.lang.Thread.State: RUNNABLE No compile task "Sweeper thread" #10 daemon prio=9 os_prio=0 cpu=109.36ms elapsed=6579.50s tid=0x00007fd1b82c8800 nid=0x8aab runnable [0x0000000000000000] java.lang.Thread.State: RUNNABLE "Common-Cleaner" #11 daemon prio=8 os_prio=0 cpu=10.62ms elapsed=6579.48s tid=0x00007fd1b8374000 nid=0x8aac in Object.wait() [0x00007fd177586000] java.lang.Thread.State: TIMED_WAITING (on object monitor) at java.lang.Object.wait(java.base@11.0.14/Native Method) - waiting on at java.lang.ref.ReferenceQueue.remove(java.base@11.0.14/ReferenceQueue.java:155) - waiting to re-lock in wait() <0x0000000777089310> (a java.lang.ref.ReferenceQueue$Lock) at jdk.internal.ref.CleanerImpl.run(java.base@11.0.14/CleanerImpl.java:148) at java.lang.Thread.run(java.base@11.0.14/Thread.java:829) at jdk.internal.misc.InnocuousThread.run(java.base@11.0.14/InnocuousThread.java:161) "org.apache.hadoop.fs.FileSystem$Statistics$StatisticsDataReferenceCleaner" #19 daemon prio=5 os_prio=0 cpu=1.15ms elapsed=6578.07s tid=0x00007fd124141000 nid=0x8ac1 in Object.wait() [0x00007fd1749fe000] java.lang.Thread.State: WAITING (on object monitor) at java.lang.Object.wait(java.base@11.0.14/Native Method) - waiting on at java.lang.ref.ReferenceQueue.remove(java.base@11.0.14/ReferenceQueue.java:155) - waiting to re-lock in wait() <0x0000000779447650> (a java.lang.ref.ReferenceQueue$Lock) at java.lang.ref.ReferenceQueue.remove(java.base@11.0.14/ReferenceQueue.java:176) at org.apache.hadoop.fs.FileSystem$Statistics$StatisticsDataReferenceCleaner.run(FileSystem.java:3762) at java.lang.Thread.run(java.base@11.0.14/Thread.java:829) "org.apache.hadoop.util.JvmPauseMonitor$Monitor@14d53139" #29 daemon prio=5 os_prio=0 cpu=1216.22ms elapsed=6577.34s tid=0x00007fd12465c800 nid=0x8ac7 waiting on condition [0x00007fd177486000] java.lang.Thread.State: TIMED_WAITING (sleeping) at java.lang.Thread.sleep(java.base@11.0.14/Native Method) at org.apache.hadoop.util.JvmPauseMonitor$Monitor.run(JvmPauseMonitor.java:192) at java.lang.Thread.run(java.base@11.0.14/Thread.java:829) "pool-5-thread-1" #30 prio=5 os_prio=0 cpu=0.17ms elapsed=6577.19s tid=0x00007fd1248c5000 nid=0x8ac9 waiting on condition [0x00007fd11f7fe000] java.lang.Thread.State: TIMED_WAITING (parking) at jdk.internal.misc.Unsafe.park(java.base@11.0.14/Native Method) - parking to wait for <0x000000077b000000> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(java.base@11.0.14/LockSupport.java:234) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(java.base@11.0.14/AbstractQueuedSynchronizer.java:2123) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(java.base@11.0.14/ScheduledThreadPoolExecutor.java:1182) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(java.base@11.0.14/ScheduledThreadPoolExecutor.java:899) at java.util.concurrent.ThreadPoolExecutor.getTask(java.base@11.0.14/ThreadPoolExecutor.java:1054) at java.util.concurrent.ThreadPoolExecutor.runWorker(java.base@11.0.14/ThreadPoolExecutor.java:1114) at java.util.concurrent.ThreadPoolExecutor$Worker.run(java.base@11.0.14/ThreadPoolExecutor.java:628) at java.lang.Thread.run(java.base@11.0.14/Thread.java:829) "qtp1645014055-31" #31 daemon prio=5 os_prio=0 cpu=0.90ms elapsed=6577.14s tid=0x00007fd124904000 nid=0x8aca runnable [0x00007fd11f310000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPoll.wait(java.base@11.0.14/Native Method) at sun.nio.ch.EPollSelectorImpl.doSelect(java.base@11.0.14/EPollSelectorImpl.java:120) at sun.nio.ch.SelectorImpl.lockAndDoSelect(java.base@11.0.14/SelectorImpl.java:124) - locked <0x000000077b01eec0> (a sun.nio.ch.Util$2) - locked <0x000000077b01ee68> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(java.base@11.0.14/SelectorImpl.java:141) at org.eclipse.jetty.io.ManagedSelector$SelectorProducer.select(ManagedSelector.java:243) at org.eclipse.jetty.io.ManagedSelector$SelectorProducer.produce(ManagedSelector.java:191) at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.executeProduceConsume(ExecuteProduceConsume.java:249) at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.produceConsume(ExecuteProduceConsume.java:148) at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.execute(ExecuteProduceConsume.java:100) at org.eclipse.jetty.io.ManagedSelector.run(ManagedSelector.java:147) at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:671) at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:589) at java.lang.Thread.run(java.base@11.0.14/Thread.java:829) "qtp1645014055-32" #32 daemon prio=5 os_prio=0 cpu=0.39ms elapsed=6577.14s tid=0x00007fd124906000 nid=0x8acb runnable [0x00007fd11f210000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPoll.wait(java.base@11.0.14/Native Method) at sun.nio.ch.EPollSelectorImpl.doSelect(java.base@11.0.14/EPollSelectorImpl.java:120) at sun.nio.ch.SelectorImpl.lockAndDoSelect(java.base@11.0.14/SelectorImpl.java:124) - locked <0x000000077b0003c0> (a sun.nio.ch.Util$2) - locked <0x000000077b000368> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(java.base@11.0.14/SelectorImpl.java:141) at org.eclipse.jetty.io.ManagedSelector$SelectorProducer.select(ManagedSelector.java:243) at org.eclipse.jetty.io.ManagedSelector$SelectorProducer.produce(ManagedSelector.java:191) at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.executeProduceConsume(ExecuteProduceConsume.java:249) at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.produceConsume(ExecuteProduceConsume.java:148) at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.execute(ExecuteProduceConsume.java:100) at org.eclipse.jetty.io.ManagedSelector.run(ManagedSelector.java:147) at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:671) at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:589) at java.lang.Thread.run(java.base@11.0.14/Thread.java:829) "qtp1645014055-33" #33 daemon prio=5 os_prio=0 cpu=0.41ms elapsed=6577.14s tid=0x00007fd124907800 nid=0x8acc runnable [0x00007fd11f110000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPoll.wait(java.base@11.0.14/Native Method) at sun.nio.ch.EPollSelectorImpl.doSelect(java.base@11.0.14/EPollSelectorImpl.java:120) at sun.nio.ch.SelectorImpl.lockAndDoSelect(java.base@11.0.14/SelectorImpl.java:124) - locked <0x000000077b0008f8> (a sun.nio.ch.Util$2) - locked <0x000000077b0008a0> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(java.base@11.0.14/SelectorImpl.java:141) at org.eclipse.jetty.io.ManagedSelector$SelectorProducer.select(ManagedSelector.java:243) at org.eclipse.jetty.io.ManagedSelector$SelectorProducer.produce(ManagedSelector.java:191) at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.executeProduceConsume(ExecuteProduceConsume.java:249) at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.produceConsume(ExecuteProduceConsume.java:148) at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.execute(ExecuteProduceConsume.java:100) at org.eclipse.jetty.io.ManagedSelector.run(ManagedSelector.java:147) at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:671) at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:589) at java.lang.Thread.run(java.base@11.0.14/Thread.java:829) "qtp1645014055-34" #34 daemon prio=5 os_prio=0 cpu=0.27ms elapsed=6577.14s tid=0x00007fd124909800 nid=0x8acd runnable [0x00007fd11f010000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPoll.wait(java.base@11.0.14/Native Method) at sun.nio.ch.EPollSelectorImpl.doSelect(java.base@11.0.14/EPollSelectorImpl.java:120) at sun.nio.ch.SelectorImpl.lockAndDoSelect(java.base@11.0.14/SelectorImpl.java:124) - locked <0x000000077b01f1b8> (a sun.nio.ch.Util$2) - locked <0x000000077b01f160> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(java.base@11.0.14/SelectorImpl.java:141) at org.eclipse.jetty.io.ManagedSelector$SelectorProducer.select(ManagedSelector.java:243) at org.eclipse.jetty.io.ManagedSelector$SelectorProducer.produce(ManagedSelector.java:191) at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.executeProduceConsume(ExecuteProduceConsume.java:249) at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.produceConsume(ExecuteProduceConsume.java:148) at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.execute(ExecuteProduceConsume.java:100) at org.eclipse.jetty.io.ManagedSelector.run(ManagedSelector.java:147) at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:671) at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:589) at java.lang.Thread.run(java.base@11.0.14/Thread.java:829) "qtp1645014055-35-acceptor-0@6e8c25a3-ServerConnector@51ca9d8b{HTTP/1.1,[http/1.1]}{zhangduo-ubuntu:35309}" #35 daemon prio=3 os_prio=0 cpu=0.60ms elapsed=6577.14s tid=0x00007fd12490b000 nid=0x8ace runnable [0x00007fd11ef10000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.ServerSocketChannelImpl.accept0(java.base@11.0.14/Native Method) at sun.nio.ch.ServerSocketChannelImpl.accept(java.base@11.0.14/ServerSocketChannelImpl.java:533) at sun.nio.ch.ServerSocketChannelImpl.accept(java.base@11.0.14/ServerSocketChannelImpl.java:285) at org.eclipse.jetty.server.ServerConnector.accept(ServerConnector.java:397) at org.eclipse.jetty.server.AbstractConnector$Acceptor.run(AbstractConnector.java:601) at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:671) at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:589) at java.lang.Thread.run(java.base@11.0.14/Thread.java:829) "qtp1645014055-36" #36 daemon prio=5 os_prio=0 cpu=7.75ms elapsed=6577.14s tid=0x00007fd12490d000 nid=0x8acf waiting on condition [0x00007fd11ee10000] java.lang.Thread.State: TIMED_WAITING (parking) at jdk.internal.misc.Unsafe.park(java.base@11.0.14/Native Method) - parking to wait for <0x000000077b01f3d8> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(java.base@11.0.14/LockSupport.java:234) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(java.base@11.0.14/AbstractQueuedSynchronizer.java:2123) at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392) at org.eclipse.jetty.util.thread.QueuedThreadPool.idleJobPoll(QueuedThreadPool.java:563) at org.eclipse.jetty.util.thread.QueuedThreadPool.access$800(QueuedThreadPool.java:48) at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:626) at java.lang.Thread.run(java.base@11.0.14/Thread.java:829) "qtp1645014055-37" #37 daemon prio=5 os_prio=0 cpu=5.66ms elapsed=6577.14s tid=0x00007fd12490f000 nid=0x8ad0 waiting on condition [0x00007fd11ed10000] java.lang.Thread.State: TIMED_WAITING (parking) at jdk.internal.misc.Unsafe.park(java.base@11.0.14/Native Method) - parking to wait for <0x000000077b01f3d8> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(java.base@11.0.14/LockSupport.java:234) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(java.base@11.0.14/AbstractQueuedSynchronizer.java:2123) at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392) at org.eclipse.jetty.util.thread.QueuedThreadPool.idleJobPoll(QueuedThreadPool.java:563) at org.eclipse.jetty.util.thread.QueuedThreadPool.access$800(QueuedThreadPool.java:48) at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:626) at java.lang.Thread.run(java.base@11.0.14/Thread.java:829) "qtp1645014055-38" #38 daemon prio=5 os_prio=0 cpu=5.30ms elapsed=6577.14s tid=0x00007fd124910800 nid=0x8ad1 waiting on condition [0x00007fd11ec10000] java.lang.Thread.State: TIMED_WAITING (parking) at jdk.internal.misc.Unsafe.park(java.base@11.0.14/Native Method) - parking to wait for <0x000000077b01f3d8> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(java.base@11.0.14/LockSupport.java:234) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(java.base@11.0.14/AbstractQueuedSynchronizer.java:2123) at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392) at org.eclipse.jetty.util.thread.QueuedThreadPool.idleJobPoll(QueuedThreadPool.java:563) at org.eclipse.jetty.util.thread.QueuedThreadPool.access$800(QueuedThreadPool.java:48) at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:626) at java.lang.Thread.run(java.base@11.0.14/Thread.java:829) "org.eclipse.jetty.server.session.HashSessionManager@7368514aTimer" #39 daemon prio=5 os_prio=0 cpu=15.38ms elapsed=6577.13s tid=0x00007fd124922000 nid=0x8ad2 waiting on condition [0x00007fd11eb10000] java.lang.Thread.State: TIMED_WAITING (parking) at jdk.internal.misc.Unsafe.park(java.base@11.0.14/Native Method) - parking to wait for <0x000000077b01f7e0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(java.base@11.0.14/LockSupport.java:234) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(java.base@11.0.14/AbstractQueuedSynchronizer.java:2123) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(java.base@11.0.14/ScheduledThreadPoolExecutor.java:1182) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(java.base@11.0.14/ScheduledThreadPoolExecutor.java:899) at java.util.concurrent.ThreadPoolExecutor.getTask(java.base@11.0.14/ThreadPoolExecutor.java:1054) at java.util.concurrent.ThreadPoolExecutor.runWorker(java.base@11.0.14/ThreadPoolExecutor.java:1114) at java.util.concurrent.ThreadPoolExecutor$Worker.run(java.base@11.0.14/ThreadPoolExecutor.java:628) at java.lang.Thread.run(java.base@11.0.14/Thread.java:829) "org.eclipse.jetty.server.session.HashSessionManager@27f5171cTimer" #40 daemon prio=5 os_prio=0 cpu=12.08ms elapsed=6577.12s tid=0x00007fd124923000 nid=0x8ad3 waiting on condition [0x00007fd11ea10000] java.lang.Thread.State: TIMED_WAITING (parking) at jdk.internal.misc.Unsafe.park(java.base@11.0.14/Native Method) - parking to wait for <0x000000077b01f8c8> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(java.base@11.0.14/LockSupport.java:234) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(java.base@11.0.14/AbstractQueuedSynchronizer.java:2123) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(java.base@11.0.14/ScheduledThreadPoolExecutor.java:1182) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(java.base@11.0.14/ScheduledThreadPoolExecutor.java:899) at java.util.concurrent.ThreadPoolExecutor.getTask(java.base@11.0.14/ThreadPoolExecutor.java:1054) at java.util.concurrent.ThreadPoolExecutor.runWorker(java.base@11.0.14/ThreadPoolExecutor.java:1114) at java.util.concurrent.ThreadPoolExecutor$Worker.run(java.base@11.0.14/ThreadPoolExecutor.java:628) at java.lang.Thread.run(java.base@11.0.14/Thread.java:829) "org.eclipse.jetty.server.session.HashSessionManager@1ec1c265Timer" #41 daemon prio=5 os_prio=0 cpu=15.62ms elapsed=6576.97s tid=0x00007fd124971800 nid=0x8ada waiting on condition [0x00007fd11e710000] java.lang.Thread.State: TIMED_WAITING (parking) at jdk.internal.misc.Unsafe.park(java.base@11.0.14/Native Method) - parking to wait for <0x000000077b0077b0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(java.base@11.0.14/LockSupport.java:234) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(java.base@11.0.14/AbstractQueuedSynchronizer.java:2123) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(java.base@11.0.14/ScheduledThreadPoolExecutor.java:1182) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(java.base@11.0.14/ScheduledThreadPoolExecutor.java:899) at java.util.concurrent.ThreadPoolExecutor.getTask(java.base@11.0.14/ThreadPoolExecutor.java:1054) at java.util.concurrent.ThreadPoolExecutor.runWorker(java.base@11.0.14/ThreadPoolExecutor.java:1114) at java.util.concurrent.ThreadPoolExecutor$Worker.run(java.base@11.0.14/ThreadPoolExecutor.java:628) at java.lang.Thread.run(java.base@11.0.14/Thread.java:829) "FSEditLogAsync" #47 daemon prio=5 os_prio=0 cpu=840592.76ms elapsed=6576.89s tid=0x00007fd1249b3800 nid=0x8adf waiting on condition [0x00007fd11e610000] java.lang.Thread.State: WAITING (parking) at jdk.internal.misc.Unsafe.park(java.base@11.0.14/Native Method) - parking to wait for <0x000000077b02ea88> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.park(java.base@11.0.14/LockSupport.java:194) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(java.base@11.0.14/AbstractQueuedSynchronizer.java:2081) at java.util.concurrent.ArrayBlockingQueue.take(java.base@11.0.14/ArrayBlockingQueue.java:417) at org.apache.hadoop.hdfs.server.namenode.FSEditLogAsync.dequeueEdit(FSEditLogAsync.java:221) at org.apache.hadoop.hdfs.server.namenode.FSEditLogAsync.run(FSEditLogAsync.java:229) at java.lang.Thread.run(java.base@11.0.14/Thread.java:829) "Socket Reader #1 for port 40133" #49 daemon prio=5 os_prio=0 cpu=1448747.20ms elapsed=6576.69s tid=0x00007fd124a96000 nid=0x8ae2 runnable [0x00007fd11e310000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPoll.wait(java.base@11.0.14/Native Method) at sun.nio.ch.EPollSelectorImpl.doSelect(java.base@11.0.14/EPollSelectorImpl.java:120) at sun.nio.ch.SelectorImpl.lockAndDoSelect(java.base@11.0.14/SelectorImpl.java:124) - locked <0x000000077b0e5538> (a sun.nio.ch.Util$2) - locked <0x000000077b0e4c80> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(java.base@11.0.14/SelectorImpl.java:141) at org.apache.hadoop.ipc.Server$Listener$Reader.doRunLoop(Server.java:1097) - locked <0x000000077a3eef20> (a org.apache.hadoop.ipc.Server$Listener$Reader) at org.apache.hadoop.ipc.Server$Listener$Reader.run(Server.java:1076) "IPC Server idle connection scanner for port 40133" #50 daemon prio=5 os_prio=0 cpu=36.36ms elapsed=6576.69s tid=0x00007fd124a9a800 nid=0x8ae3 in Object.wait() [0x00007fd11e210000] java.lang.Thread.State: TIMED_WAITING (on object monitor) at java.lang.Object.wait(java.base@11.0.14/Native Method) - waiting on at java.util.TimerThread.mainLoop(java.base@11.0.14/Timer.java:553) - waiting to re-lock in wait() <0x000000077b02ecd0> (a java.util.TaskQueue) at java.util.TimerThread.run(java.base@11.0.14/Timer.java:506) "org.apache.hadoop.hdfs.server.blockmanagement.PendingReconstructionBlocks$PendingReconstructionMonitor@4ea833eb" #54 daemon prio=5 os_prio=0 cpu=1.15ms elapsed=6576.48s tid=0x00007fd124bb5000 nid=0x8af0 waiting on condition [0x00007fd11fffe000] java.lang.Thread.State: TIMED_WAITING (sleeping) at java.lang.Thread.sleep(java.base@11.0.14/Native Method) at org.apache.hadoop.hdfs.server.blockmanagement.PendingReconstructionBlocks$PendingReconstructionMonitor.run(PendingReconstructionBlocks.java:246) at java.lang.Thread.run(java.base@11.0.14/Thread.java:829) "DatanodeAdminMonitor-0" #55 daemon prio=5 os_prio=0 cpu=142.04ms elapsed=6576.47s tid=0x00007fd124bbc000 nid=0x8af1 waiting on condition [0x00007fd11f610000] java.lang.Thread.State: TIMED_WAITING (parking) at jdk.internal.misc.Unsafe.park(java.base@11.0.14/Native Method) - parking to wait for <0x000000077b02ed68> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(java.base@11.0.14/LockSupport.java:234) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(java.base@11.0.14/AbstractQueuedSynchronizer.java:2123) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(java.base@11.0.14/ScheduledThreadPoolExecutor.java:1182) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(java.base@11.0.14/ScheduledThreadPoolExecutor.java:899) at java.util.concurrent.ThreadPoolExecutor.getTask(java.base@11.0.14/ThreadPoolExecutor.java:1054) at java.util.concurrent.ThreadPoolExecutor.runWorker(java.base@11.0.14/ThreadPoolExecutor.java:1114) at java.util.concurrent.ThreadPoolExecutor$Worker.run(java.base@11.0.14/ThreadPoolExecutor.java:628) at java.lang.Thread.run(java.base@11.0.14/Thread.java:829) "org.apache.hadoop.hdfs.server.blockmanagement.HeartbeatManager$Monitor@10032eba" #45 daemon prio=5 os_prio=0 cpu=59.26ms elapsed=6576.47s tid=0x00007fd124bbd000 nid=0x8af2 waiting on condition [0x00007fd11dd10000] java.lang.Thread.State: TIMED_WAITING (sleeping) at java.lang.Thread.sleep(java.base@11.0.14/Native Method) at org.apache.hadoop.hdfs.server.blockmanagement.HeartbeatManager$Monitor.run(HeartbeatManager.java:456) at java.lang.Thread.run(java.base@11.0.14/Thread.java:829) "RedundancyMonitor" #42 daemon prio=5 os_prio=0 cpu=177.53ms elapsed=6576.47s tid=0x00007fd124bbf000 nid=0x8af3 waiting on condition [0x00007fd11dc10000] java.lang.Thread.State: TIMED_WAITING (sleeping) at java.lang.Thread.sleep(java.base@11.0.14/Native Method) at java.lang.Thread.sleep(java.base@11.0.14/Thread.java:334) at java.util.concurrent.TimeUnit.sleep(java.base@11.0.14/TimeUnit.java:446) at org.apache.hadoop.hdfs.server.blockmanagement.BlockManager$RedundancyMonitor.run(BlockManager.java:4557) at java.lang.Thread.run(java.base@11.0.14/Thread.java:829) "StorageInfoMonitor" #43 daemon prio=5 os_prio=0 cpu=2.12ms elapsed=6576.47s tid=0x00007fd124bc0800 nid=0x8af4 waiting on condition [0x00007fd11db10000] java.lang.Thread.State: TIMED_WAITING (sleeping) at java.lang.Thread.sleep(java.base@11.0.14/Native Method) at org.apache.hadoop.hdfs.server.blockmanagement.BlockManager$StorageInfoDefragmenter.run(BlockManager.java:4592) at java.lang.Thread.run(java.base@11.0.14/Thread.java:829) "Block report processor" #44 daemon prio=5 os_prio=0 cpu=611.03ms elapsed=6576.47s tid=0x00007fd124bc2800 nid=0x8af5 waiting on condition [0x00007fd11da10000] java.lang.Thread.State: WAITING (parking) at jdk.internal.misc.Unsafe.park(java.base@11.0.14/Native Method) - parking to wait for <0x000000077a3ef250> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.park(java.base@11.0.14/LockSupport.java:194) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(java.base@11.0.14/AbstractQueuedSynchronizer.java:2081) at java.util.concurrent.ArrayBlockingQueue.take(java.base@11.0.14/ArrayBlockingQueue.java:417) at org.apache.hadoop.hdfs.server.blockmanagement.BlockManager$BlockReportProcessingThread.processQueue(BlockManager.java:4916) at org.apache.hadoop.hdfs.server.blockmanagement.BlockManager$BlockReportProcessingThread.run(BlockManager.java:4905) "IPC Server Responder" #51 daemon prio=5 os_prio=0 cpu=0.71ms elapsed=6576.44s tid=0x00007fd124bda000 nid=0x8af7 runnable [0x00007fd11d910000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPoll.wait(java.base@11.0.14/Native Method) at sun.nio.ch.EPollSelectorImpl.doSelect(java.base@11.0.14/EPollSelectorImpl.java:120) at sun.nio.ch.SelectorImpl.lockAndDoSelect(java.base@11.0.14/SelectorImpl.java:124) - locked <0x000000077b05e178> (a sun.nio.ch.Util$2) - locked <0x000000077b05e120> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(java.base@11.0.14/SelectorImpl.java:136) at org.apache.hadoop.ipc.Server$Responder.doRunLoop(Server.java:1334) at org.apache.hadoop.ipc.Server$Responder.run(Server.java:1317) "IPC Server listener on 40133" #48 daemon prio=5 os_prio=0 cpu=3.30ms elapsed=6576.44s tid=0x00007fd124bdb000 nid=0x8af8 runnable [0x00007fd11d810000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPoll.wait(java.base@11.0.14/Native Method) at sun.nio.ch.EPollSelectorImpl.doSelect(java.base@11.0.14/EPollSelectorImpl.java:120) at sun.nio.ch.SelectorImpl.lockAndDoSelect(java.base@11.0.14/SelectorImpl.java:124) - locked <0x000000077b05e3e8> (a sun.nio.ch.Util$2) - locked <0x000000077b05e390> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(java.base@11.0.14/SelectorImpl.java:141) at org.apache.hadoop.ipc.Server$Listener.run(Server.java:1159) "IPC Server handler 0 on 40133" #57 daemon prio=5 os_prio=0 cpu=494851.63ms elapsed=6576.44s tid=0x00007fd124bdc800 nid=0x8af9 runnable [0x00007fd11d70f000] java.lang.Thread.State: RUNNABLE at java.util.ResourceBundle.getString(java.base@11.0.14/ResourceBundle.java:521) at sun.security.util.ResourcesMgr.getString(java.base@11.0.14/ResourcesMgr.java:40) at javax.security.auth.Subject.doAs(java.base@11.0.14/Subject.java:416) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1730) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:2682) "IPC Server handler 1 on 40133" #58 daemon prio=5 os_prio=0 cpu=494683.85ms elapsed=6576.44s tid=0x00007fd124bde800 nid=0x8afa waiting on condition [0x00007fd11d610000] java.lang.Thread.State: TIMED_WAITING (parking) at jdk.internal.misc.Unsafe.park(java.base@11.0.14/Native Method) - parking to wait for <0x000000077b05e5e0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(java.base@11.0.14/LockSupport.java:234) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(java.base@11.0.14/AbstractQueuedSynchronizer.java:2123) at java.util.concurrent.LinkedBlockingQueue.poll(java.base@11.0.14/LinkedBlockingQueue.java:458) at org.apache.hadoop.ipc.CallQueueManager.take(CallQueueManager.java:288) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:2668) "IPC Server handler 2 on 40133" #59 daemon prio=5 os_prio=0 cpu=494570.82ms elapsed=6576.44s tid=0x00007fd124be0800 nid=0x8afb waiting on condition [0x00007fd11d510000] java.lang.Thread.State: TIMED_WAITING (parking) at jdk.internal.misc.Unsafe.park(java.base@11.0.14/Native Method) - parking to wait for <0x000000077b05e5e0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(java.base@11.0.14/LockSupport.java:234) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(java.base@11.0.14/AbstractQueuedSynchronizer.java:2123) at java.util.concurrent.LinkedBlockingQueue.poll(java.base@11.0.14/LinkedBlockingQueue.java:458) at org.apache.hadoop.ipc.CallQueueManager.take(CallQueueManager.java:288) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:2668) "IPC Server handler 3 on 40133" #60 daemon prio=5 os_prio=0 cpu=494859.22ms elapsed=6576.44s tid=0x00007fd124be2800 nid=0x8afc waiting on condition [0x00007fd11d410000] java.lang.Thread.State: TIMED_WAITING (parking) at jdk.internal.misc.Unsafe.park(java.base@11.0.14/Native Method) - parking to wait for <0x000000077b05e5e0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(java.base@11.0.14/LockSupport.java:234) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(java.base@11.0.14/AbstractQueuedSynchronizer.java:2123) at java.util.concurrent.LinkedBlockingQueue.poll(java.base@11.0.14/LinkedBlockingQueue.java:458) at org.apache.hadoop.ipc.CallQueueManager.take(CallQueueManager.java:288) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:2668) "IPC Server handler 4 on 40133" #61 daemon prio=5 os_prio=0 cpu=494157.11ms elapsed=6576.44s tid=0x00007fd124be4000 nid=0x8afd waiting on condition [0x00007fd11d310000] java.lang.Thread.State: TIMED_WAITING (parking) at jdk.internal.misc.Unsafe.park(java.base@11.0.14/Native Method) - parking to wait for <0x000000077b05e5e0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(java.base@11.0.14/LockSupport.java:234) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(java.base@11.0.14/AbstractQueuedSynchronizer.java:2123) at java.util.concurrent.LinkedBlockingQueue.poll(java.base@11.0.14/LinkedBlockingQueue.java:458) at org.apache.hadoop.ipc.CallQueueManager.take(CallQueueManager.java:288) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:2668) "pool-7-thread-1" #62 prio=5 os_prio=0 cpu=636.26ms elapsed=6576.44s tid=0x00007fd124be6800 nid=0x8afe waiting on condition [0x00007fd11d210000] java.lang.Thread.State: TIMED_WAITING (parking) at jdk.internal.misc.Unsafe.park(java.base@11.0.14/Native Method) - parking to wait for <0x000000077b0ee890> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(java.base@11.0.14/LockSupport.java:234) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(java.base@11.0.14/AbstractQueuedSynchronizer.java:2123) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(java.base@11.0.14/ScheduledThreadPoolExecutor.java:1182) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(java.base@11.0.14/ScheduledThreadPoolExecutor.java:899) at java.util.concurrent.ThreadPoolExecutor.getTask(java.base@11.0.14/ThreadPoolExecutor.java:1054) at java.util.concurrent.ThreadPoolExecutor.runWorker(java.base@11.0.14/ThreadPoolExecutor.java:1114) at java.util.concurrent.ThreadPoolExecutor$Worker.run(java.base@11.0.14/ThreadPoolExecutor.java:628) at java.lang.Thread.run(java.base@11.0.14/Thread.java:829) "org.apache.hadoop.hdfs.server.namenode.LeaseManager$Monitor@3951fb8f" #64 daemon prio=5 os_prio=0 cpu=171.98ms elapsed=6576.43s tid=0x00007fd124be8800 nid=0x8b00 waiting on condition [0x00007fd11d110000] java.lang.Thread.State: TIMED_WAITING (sleeping) at java.lang.Thread.sleep(java.base@11.0.14/Native Method) at org.apache.hadoop.hdfs.server.namenode.LeaseManager$Monitor.run(LeaseManager.java:534) at java.lang.Thread.run(java.base@11.0.14/Thread.java:829) "org.apache.hadoop.hdfs.server.namenode.FSNamesystem$NameNodeResourceMonitor@504ba335" #65 daemon prio=5 os_prio=0 cpu=95.92ms elapsed=6576.43s tid=0x00007fd124beb000 nid=0x8b01 waiting on condition [0x00007fd11d010000] java.lang.Thread.State: TIMED_WAITING (sleeping) at java.lang.Thread.sleep(java.base@11.0.14/Native Method) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem$NameNodeResourceMonitor.run(FSNamesystem.java:4045) at java.lang.Thread.run(java.base@11.0.14/Thread.java:829) "org.apache.hadoop.hdfs.server.namenode.FSNamesystem$NameNodeEditLogRoller@3a6fa2fd" #66 daemon prio=5 os_prio=0 cpu=26.67ms elapsed=6576.43s tid=0x00007fd124bed000 nid=0x8b02 waiting on condition [0x00007fd11cf10000] java.lang.Thread.State: TIMED_WAITING (sleeping) at java.lang.Thread.sleep(java.base@11.0.14/Native Method) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem$NameNodeEditLogRoller.run(FSNamesystem.java:4087) at java.lang.Thread.run(java.base@11.0.14/Thread.java:829) "org.apache.hadoop.hdfs.server.namenode.FSNamesystem$LazyPersistFileScrubber@4be6e6ec" #67 daemon prio=5 os_prio=0 cpu=1.52ms elapsed=6576.43s tid=0x00007fd124bee800 nid=0x8b03 waiting on condition [0x00007fd11ce10000] java.lang.Thread.State: TIMED_WAITING (sleeping) at java.lang.Thread.sleep(java.base@11.0.14/Native Method) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem$LazyPersistFileScrubber.run(FSNamesystem.java:4178) at java.lang.Thread.run(java.base@11.0.14/Thread.java:829) "CacheReplicationMonitor(1829512334)" #68 daemon prio=5 os_prio=0 cpu=87.27ms elapsed=6576.43s tid=0x00007fd124bf0800 nid=0x8b04 waiting on condition [0x00007fd11cd10000] java.lang.Thread.State: TIMED_WAITING (parking) at jdk.internal.misc.Unsafe.park(java.base@11.0.14/Native Method) - parking to wait for <0x000000077b082eb8> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(java.base@11.0.14/LockSupport.java:234) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(java.base@11.0.14/AbstractQueuedSynchronizer.java:2211) at org.apache.hadoop.hdfs.server.blockmanagement.CacheReplicationMonitor.run(CacheReplicationMonitor.java:181) "pool-12-thread-1" #78 prio=5 os_prio=0 cpu=0.13ms elapsed=6576.36s tid=0x00007fd124c4d000 nid=0x8b10 waiting on condition [0x00007fd11cb10000] java.lang.Thread.State: TIMED_WAITING (parking) at jdk.internal.misc.Unsafe.park(java.base@11.0.14/Native Method) - parking to wait for <0x000000077b096d28> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(java.base@11.0.14/LockSupport.java:234) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(java.base@11.0.14/AbstractQueuedSynchronizer.java:2123) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(java.base@11.0.14/ScheduledThreadPoolExecutor.java:1182) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(java.base@11.0.14/ScheduledThreadPoolExecutor.java:899) at java.util.concurrent.ThreadPoolExecutor.getTask(java.base@11.0.14/ThreadPoolExecutor.java:1054) at java.util.concurrent.ThreadPoolExecutor.runWorker(java.base@11.0.14/ThreadPoolExecutor.java:1114) at java.util.concurrent.ThreadPoolExecutor$Worker.run(java.base@11.0.14/ThreadPoolExecutor.java:628) at java.lang.Thread.run(java.base@11.0.14/Thread.java:829) "qtp753180475-79" #79 daemon prio=5 os_prio=0 cpu=0.35ms elapsed=6576.36s tid=0x00007fd124c4f800 nid=0x8b12 runnable [0x00007fd11c810000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPoll.wait(java.base@11.0.14/Native Method) at sun.nio.ch.EPollSelectorImpl.doSelect(java.base@11.0.14/EPollSelectorImpl.java:120) at sun.nio.ch.SelectorImpl.lockAndDoSelect(java.base@11.0.14/SelectorImpl.java:124) - locked <0x000000077b096fd8> (a sun.nio.ch.Util$2) - locked <0x000000077b096f80> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(java.base@11.0.14/SelectorImpl.java:141) at org.eclipse.jetty.io.ManagedSelector$SelectorProducer.select(ManagedSelector.java:243) at org.eclipse.jetty.io.ManagedSelector$SelectorProducer.produce(ManagedSelector.java:191) at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.executeProduceConsume(ExecuteProduceConsume.java:249) at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.produceConsume(ExecuteProduceConsume.java:148) at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.execute(ExecuteProduceConsume.java:100) at org.eclipse.jetty.io.ManagedSelector.run(ManagedSelector.java:147) at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:671) at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:589) at java.lang.Thread.run(java.base@11.0.14/Thread.java:829) "qtp753180475-80-acceptor-0@7125c3df-ServerConnector@528b4d11{HTTP/1.1,[http/1.1]}{localhost:43315}" #80 daemon prio=3 os_prio=0 cpu=1.23ms elapsed=6576.35s tid=0x00007fd124c51000 nid=0x8b13 runnable [0x00007fd11c710000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.ServerSocketChannelImpl.accept0(java.base@11.0.14/Native Method) at sun.nio.ch.ServerSocketChannelImpl.accept(java.base@11.0.14/ServerSocketChannelImpl.java:533) at sun.nio.ch.ServerSocketChannelImpl.accept(java.base@11.0.14/ServerSocketChannelImpl.java:285) at org.eclipse.jetty.server.ServerConnector.accept(ServerConnector.java:397) at org.eclipse.jetty.server.AbstractConnector$Acceptor.run(AbstractConnector.java:601) at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:671) at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:589) at java.lang.Thread.run(java.base@11.0.14/Thread.java:829) "qtp753180475-81" #81 daemon prio=5 os_prio=0 cpu=5.56ms elapsed=6576.35s tid=0x00007fd124c56800 nid=0x8b14 waiting on condition [0x00007fd11c610000] java.lang.Thread.State: TIMED_WAITING (parking) at jdk.internal.misc.Unsafe.park(java.base@11.0.14/Native Method) - parking to wait for <0x000000077b0976c0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(java.base@11.0.14/LockSupport.java:234) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(java.base@11.0.14/AbstractQueuedSynchronizer.java:2123) at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392) at org.eclipse.jetty.util.thread.QueuedThreadPool.idleJobPoll(QueuedThreadPool.java:563) at org.eclipse.jetty.util.thread.QueuedThreadPool.access$800(QueuedThreadPool.java:48) at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:626) at java.lang.Thread.run(java.base@11.0.14/Thread.java:829) "org.eclipse.jetty.server.session.HashSessionManager@64671f5aTimer" #82 daemon prio=5 os_prio=0 cpu=12.56ms elapsed=6576.35s tid=0x00007fd124c58000 nid=0x8b15 waiting on condition [0x00007fd11c510000] java.lang.Thread.State: TIMED_WAITING (parking) at jdk.internal.misc.Unsafe.park(java.base@11.0.14/Native Method) - parking to wait for <0x000000077b0ef668> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(java.base@11.0.14/LockSupport.java:234) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(java.base@11.0.14/AbstractQueuedSynchronizer.java:2123) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(java.base@11.0.14/ScheduledThreadPoolExecutor.java:1182) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(java.base@11.0.14/ScheduledThreadPoolExecutor.java:899) at java.util.concurrent.ThreadPoolExecutor.getTask(java.base@11.0.14/ThreadPoolExecutor.java:1054) at java.util.concurrent.ThreadPoolExecutor.runWorker(java.base@11.0.14/ThreadPoolExecutor.java:1114) at java.util.concurrent.ThreadPoolExecutor$Worker.run(java.base@11.0.14/ThreadPoolExecutor.java:628) at java.lang.Thread.run(java.base@11.0.14/Thread.java:829) "org.eclipse.jetty.server.session.HashSessionManager@21749aa3Timer" #83 daemon prio=5 os_prio=0 cpu=12.46ms elapsed=6576.35s tid=0x00007fd124c5a000 nid=0x8b16 waiting on condition [0x00007fd11c410000] java.lang.Thread.State: TIMED_WAITING (parking) at jdk.internal.misc.Unsafe.park(java.base@11.0.14/Native Method) - parking to wait for <0x000000077b0ef750> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(java.base@11.0.14/LockSupport.java:234) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(java.base@11.0.14/AbstractQueuedSynchronizer.java:2123) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(java.base@11.0.14/ScheduledThreadPoolExecutor.java:1182) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(java.base@11.0.14/ScheduledThreadPoolExecutor.java:899) at java.util.concurrent.ThreadPoolExecutor.getTask(java.base@11.0.14/ThreadPoolExecutor.java:1054) at java.util.concurrent.ThreadPoolExecutor.runWorker(java.base@11.0.14/ThreadPoolExecutor.java:1114) at java.util.concurrent.ThreadPoolExecutor$Worker.run(java.base@11.0.14/ThreadPoolExecutor.java:628) at java.lang.Thread.run(java.base@11.0.14/Thread.java:829) "org.eclipse.jetty.server.session.HashSessionManager@52ae9bbfTimer" #84 daemon prio=5 os_prio=0 cpu=13.94ms elapsed=6576.27s tid=0x00007fd124c35000 nid=0x8b18 waiting on condition [0x00007fd11c210000] java.lang.Thread.State: TIMED_WAITING (parking) at jdk.internal.misc.Unsafe.park(java.base@11.0.14/Native Method) - parking to wait for <0x000000077b0ef838> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(java.base@11.0.14/LockSupport.java:234) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(java.base@11.0.14/AbstractQueuedSynchronizer.java:2123) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(java.base@11.0.14/ScheduledThreadPoolExecutor.java:1182) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(java.base@11.0.14/ScheduledThreadPoolExecutor.java:899) at java.util.concurrent.ThreadPoolExecutor.getTask(java.base@11.0.14/ThreadPoolExecutor.java:1054) at java.util.concurrent.ThreadPoolExecutor.runWorker(java.base@11.0.14/ThreadPoolExecutor.java:1114) at java.util.concurrent.ThreadPoolExecutor$Worker.run(java.base@11.0.14/ThreadPoolExecutor.java:628) at java.lang.Thread.run(java.base@11.0.14/Thread.java:829) "nioEventLoopGroup-2-1" #85 prio=10 os_prio=0 cpu=5.14ms elapsed=6576.15s tid=0x00007fd124d1b000 nid=0x8b19 runnable [0x00007fd11c110000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPoll.wait(java.base@11.0.14/Native Method) at sun.nio.ch.EPollSelectorImpl.doSelect(java.base@11.0.14/EPollSelectorImpl.java:120) at sun.nio.ch.SelectorImpl.lockAndDoSelect(java.base@11.0.14/SelectorImpl.java:124) - locked <0x000000077b180888> (a io.netty.channel.nio.SelectedSelectionKeySet) - locked <0x000000077b180830> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(java.base@11.0.14/SelectorImpl.java:141) at io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) at io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:803) at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:457) at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:989) at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) at io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) at java.lang.Thread.run(java.base@11.0.14/Thread.java:829) "org.apache.hadoop.util.JvmPauseMonitor$Monitor@2eba923c" #86 daemon prio=5 os_prio=0 cpu=1232.47ms elapsed=6576.14s tid=0x00007fd124d17800 nid=0x8b1a waiting on condition [0x00007fd09bdfe000] java.lang.Thread.State: TIMED_WAITING (sleeping) at java.lang.Thread.sleep(java.base@11.0.14/Native Method) at org.apache.hadoop.util.JvmPauseMonitor$Monitor.run(JvmPauseMonitor.java:192) at java.lang.Thread.run(java.base@11.0.14/Thread.java:829) "Socket Reader #1 for port 45509" #88 daemon prio=5 os_prio=0 cpu=0.29ms elapsed=6576.10s tid=0x00007fd124d29800 nid=0x8b1b runnable [0x00007fd09bafe000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPoll.wait(java.base@11.0.14/Native Method) at sun.nio.ch.EPollSelectorImpl.doSelect(java.base@11.0.14/EPollSelectorImpl.java:120) at sun.nio.ch.SelectorImpl.lockAndDoSelect(java.base@11.0.14/SelectorImpl.java:124) - locked <0x000000077b238280> (a sun.nio.ch.Util$2) - locked <0x000000077b238228> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(java.base@11.0.14/SelectorImpl.java:141) at org.apache.hadoop.ipc.Server$Listener$Reader.doRunLoop(Server.java:1097) - locked <0x000000077a3eeda0> (a org.apache.hadoop.ipc.Server$Listener$Reader) at org.apache.hadoop.ipc.Server$Listener$Reader.run(Server.java:1076) "IPC Server idle connection scanner for port 45509" #89 daemon prio=5 os_prio=0 cpu=40.62ms elapsed=6576.10s tid=0x00007fd124d2b000 nid=0x8b1c in Object.wait() [0x00007fd09b9fe000] java.lang.Thread.State: TIMED_WAITING (on object monitor) at java.lang.Object.wait(java.base@11.0.14/Native Method) - waiting on at java.util.TimerThread.mainLoop(java.base@11.0.14/Timer.java:553) - waiting to re-lock in wait() <0x000000077b214068> (a java.util.TaskQueue) at java.util.TimerThread.run(java.base@11.0.14/Timer.java:506) "BP-1865003422-127.0.0.1-1652709596755 heartbeating to zhangduo-ubuntu/127.0.0.1:40133" #93 daemon prio=5 os_prio=0 cpu=1790.31ms elapsed=6576.07s tid=0x00007fd124d4d000 nid=0x8b1e in Object.wait() [0x00007fd09b8fe000] java.lang.Thread.State: TIMED_WAITING (on object monitor) at java.lang.Object.wait(java.base@11.0.14/Native Method) - waiting on at org.apache.hadoop.hdfs.server.datanode.IncrementalBlockReportManager.waitTillNextIBR(IncrementalBlockReportManager.java:158) - waiting to re-lock in wait() <0x000000077b1abf98> (a org.apache.hadoop.hdfs.server.datanode.IncrementalBlockReportManager) at org.apache.hadoop.hdfs.server.datanode.BPServiceActor.offerService(BPServiceActor.java:717) at org.apache.hadoop.hdfs.server.datanode.BPServiceActor.run(BPServiceActor.java:844) at java.lang.Thread.run(java.base@11.0.14/Thread.java:829) "pool-13-thread-1" #94 prio=5 os_prio=0 cpu=381.01ms elapsed=6576.07s tid=0x00007fd124d50000 nid=0x8b1f waiting on condition [0x00007fd09b7fe000] java.lang.Thread.State: TIMED_WAITING (parking) at jdk.internal.misc.Unsafe.park(java.base@11.0.14/Native Method) - parking to wait for <0x000000077b218208> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(java.base@11.0.14/LockSupport.java:234) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(java.base@11.0.14/AbstractQueuedSynchronizer.java:2123) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(java.base@11.0.14/ScheduledThreadPoolExecutor.java:1182) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(java.base@11.0.14/ScheduledThreadPoolExecutor.java:899) at java.util.concurrent.ThreadPoolExecutor.getTask(java.base@11.0.14/ThreadPoolExecutor.java:1054) at java.util.concurrent.ThreadPoolExecutor.runWorker(java.base@11.0.14/ThreadPoolExecutor.java:1114) at java.util.concurrent.ThreadPoolExecutor$Worker.run(java.base@11.0.14/ThreadPoolExecutor.java:628) at java.lang.Thread.run(java.base@11.0.14/Thread.java:829) "org.apache.hadoop.hdfs.server.datanode.DataXceiverServer@46131258" #77 daemon prio=5 os_prio=0 cpu=1098.93ms elapsed=6576.06s tid=0x00007fd124d51800 nid=0x8b20 runnable [0x00007fd09b6fe000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.ServerSocketChannelImpl.accept0(java.base@11.0.14/Native Method) at sun.nio.ch.ServerSocketChannelImpl.accept(java.base@11.0.14/ServerSocketChannelImpl.java:533) at sun.nio.ch.ServerSocketChannelImpl.accept(java.base@11.0.14/ServerSocketChannelImpl.java:285) at sun.nio.ch.ServerSocketAdaptor.accept(java.base@11.0.14/ServerSocketAdaptor.java:114) - locked <0x000000077b20f410> (a java.lang.Object) at org.apache.hadoop.hdfs.net.TcpPeerServer.accept(TcpPeerServer.java:85) at org.apache.hadoop.hdfs.server.datanode.DataXceiverServer.run(DataXceiverServer.java:145) at java.lang.Thread.run(java.base@11.0.14/Thread.java:829) "IPC Server Responder" #90 daemon prio=5 os_prio=0 cpu=0.71ms elapsed=6576.06s tid=0x00007fd124d54800 nid=0x8b21 runnable [0x00007fd09b5fe000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPoll.wait(java.base@11.0.14/Native Method) at sun.nio.ch.EPollSelectorImpl.doSelect(java.base@11.0.14/EPollSelectorImpl.java:120) at sun.nio.ch.SelectorImpl.lockAndDoSelect(java.base@11.0.14/SelectorImpl.java:124) - locked <0x000000077b215138> (a sun.nio.ch.Util$2) - locked <0x000000077b214fe8> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(java.base@11.0.14/SelectorImpl.java:136) at org.apache.hadoop.ipc.Server$Responder.doRunLoop(Server.java:1334) at org.apache.hadoop.ipc.Server$Responder.run(Server.java:1317) "IPC Server listener on 45509" #87 daemon prio=5 os_prio=0 cpu=0.44ms elapsed=6576.06s tid=0x00007fd124d56000 nid=0x8b22 runnable [0x00007fd09b4fe000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPoll.wait(java.base@11.0.14/Native Method) at sun.nio.ch.EPollSelectorImpl.doSelect(java.base@11.0.14/EPollSelectorImpl.java:120) at sun.nio.ch.SelectorImpl.lockAndDoSelect(java.base@11.0.14/SelectorImpl.java:124) - locked <0x000000077b214be0> (a sun.nio.ch.Util$2) - locked <0x000000077b2147e0> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(java.base@11.0.14/SelectorImpl.java:141) at org.apache.hadoop.ipc.Server$Listener.run(Server.java:1159) "IPC Server handler 0 on 45509" #95 daemon prio=5 os_prio=0 cpu=228.26ms elapsed=6576.06s tid=0x00007fd124d58000 nid=0x8b23 waiting on condition [0x00007fd09b3fe000] java.lang.Thread.State: TIMED_WAITING (parking) at jdk.internal.misc.Unsafe.park(java.base@11.0.14/Native Method) - parking to wait for <0x000000077b213138> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(java.base@11.0.14/LockSupport.java:234) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(java.base@11.0.14/AbstractQueuedSynchronizer.java:2123) at java.util.concurrent.LinkedBlockingQueue.poll(java.base@11.0.14/LinkedBlockingQueue.java:458) at org.apache.hadoop.ipc.CallQueueManager.take(CallQueueManager.java:288) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:2668) "IPC Server handler 1 on 45509" #96 daemon prio=5 os_prio=0 cpu=234.98ms elapsed=6576.06s tid=0x00007fd124d5a000 nid=0x8b24 waiting on condition [0x00007fd09b2fe000] java.lang.Thread.State: TIMED_WAITING (parking) at jdk.internal.misc.Unsafe.park(java.base@11.0.14/Native Method) - parking to wait for <0x000000077b213138> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(java.base@11.0.14/LockSupport.java:234) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(java.base@11.0.14/AbstractQueuedSynchronizer.java:2123) at java.util.concurrent.LinkedBlockingQueue.poll(java.base@11.0.14/LinkedBlockingQueue.java:458) at org.apache.hadoop.ipc.CallQueueManager.take(CallQueueManager.java:288) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:2668) "IPC Server handler 2 on 45509" #97 daemon prio=5 os_prio=0 cpu=231.48ms elapsed=6576.06s tid=0x00007fd124d5b800 nid=0x8b25 waiting on condition [0x00007fd09b1fe000] java.lang.Thread.State: TIMED_WAITING (parking) at jdk.internal.misc.Unsafe.park(java.base@11.0.14/Native Method) - parking to wait for <0x000000077b213138> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(java.base@11.0.14/LockSupport.java:234) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(java.base@11.0.14/AbstractQueuedSynchronizer.java:2123) at java.util.concurrent.LinkedBlockingQueue.poll(java.base@11.0.14/LinkedBlockingQueue.java:458) at org.apache.hadoop.ipc.CallQueueManager.take(CallQueueManager.java:288) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:2668) "IPC Server handler 3 on 45509" #98 daemon prio=5 os_prio=0 cpu=230.76ms elapsed=6576.06s tid=0x00007fd124d5d800 nid=0x8b26 waiting on condition [0x00007fd09b0fe000] java.lang.Thread.State: TIMED_WAITING (parking) at jdk.internal.misc.Unsafe.park(java.base@11.0.14/Native Method) - parking to wait for <0x000000077b213138> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(java.base@11.0.14/LockSupport.java:234) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(java.base@11.0.14/AbstractQueuedSynchronizer.java:2123) at java.util.concurrent.LinkedBlockingQueue.poll(java.base@11.0.14/LinkedBlockingQueue.java:458) at org.apache.hadoop.ipc.CallQueueManager.take(CallQueueManager.java:288) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:2668) "IPC Server handler 4 on 45509" #99 daemon prio=5 os_prio=0 cpu=232.72ms elapsed=6576.06s tid=0x00007fd124d5f000 nid=0x8b27 waiting on condition [0x00007fd09affe000] java.lang.Thread.State: TIMED_WAITING (parking) at jdk.internal.misc.Unsafe.park(java.base@11.0.14/Native Method) - parking to wait for <0x000000077b213138> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(java.base@11.0.14/LockSupport.java:234) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(java.base@11.0.14/AbstractQueuedSynchronizer.java:2123) at java.util.concurrent.LinkedBlockingQueue.poll(java.base@11.0.14/LinkedBlockingQueue.java:458) at org.apache.hadoop.ipc.CallQueueManager.take(CallQueueManager.java:288) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:2668) "pool-19-thread-1" #108 prio=5 os_prio=0 cpu=0.15ms elapsed=6576.04s tid=0x00007fd124d65800 nid=0x8b32 waiting on condition [0x00007fd09acfe000] java.lang.Thread.State: TIMED_WAITING (parking) at jdk.internal.misc.Unsafe.park(java.base@11.0.14/Native Method) - parking to wait for <0x000000077b238638> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(java.base@11.0.14/LockSupport.java:234) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(java.base@11.0.14/AbstractQueuedSynchronizer.java:2123) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(java.base@11.0.14/ScheduledThreadPoolExecutor.java:1182) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(java.base@11.0.14/ScheduledThreadPoolExecutor.java:899) at java.util.concurrent.ThreadPoolExecutor.getTask(java.base@11.0.14/ThreadPoolExecutor.java:1054) at java.util.concurrent.ThreadPoolExecutor.runWorker(java.base@11.0.14/ThreadPoolExecutor.java:1114) at java.util.concurrent.ThreadPoolExecutor$Worker.run(java.base@11.0.14/ThreadPoolExecutor.java:628) at java.lang.Thread.run(java.base@11.0.14/Thread.java:829) "qtp1552055888-109" #109 daemon prio=5 os_prio=0 cpu=0.28ms elapsed=6576.04s tid=0x00007fd124d67000 nid=0x8b33 runnable [0x00007fd09abfe000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPoll.wait(java.base@11.0.14/Native Method) at sun.nio.ch.EPollSelectorImpl.doSelect(java.base@11.0.14/EPollSelectorImpl.java:120) at sun.nio.ch.SelectorImpl.lockAndDoSelect(java.base@11.0.14/SelectorImpl.java:124) - locked <0x000000077b21de90> (a sun.nio.ch.Util$2) - locked <0x000000077b21de38> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(java.base@11.0.14/SelectorImpl.java:141) at org.eclipse.jetty.io.ManagedSelector$SelectorProducer.select(ManagedSelector.java:243) at org.eclipse.jetty.io.ManagedSelector$SelectorProducer.produce(ManagedSelector.java:191) at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.produceExecuteConsume(ExecuteProduceConsume.java:169) at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.produceConsume(ExecuteProduceConsume.java:145) at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.execute(ExecuteProduceConsume.java:100) at org.eclipse.jetty.io.ManagedSelector.run(ManagedSelector.java:147) at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:671) at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:589) at java.lang.Thread.run(java.base@11.0.14/Thread.java:829) "qtp1552055888-110-acceptor-0@63e8ba39-ServerConnector@240b5e63{HTTP/1.1,[http/1.1]}{localhost:33789}" #110 daemon prio=3 os_prio=0 cpu=0.51ms elapsed=6576.04s tid=0x00007fd124d68800 nid=0x8b34 runnable [0x00007fd09aafe000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.ServerSocketChannelImpl.accept0(java.base@11.0.14/Native Method) at sun.nio.ch.ServerSocketChannelImpl.accept(java.base@11.0.14/ServerSocketChannelImpl.java:533) at sun.nio.ch.ServerSocketChannelImpl.accept(java.base@11.0.14/ServerSocketChannelImpl.java:285) at org.eclipse.jetty.server.ServerConnector.accept(ServerConnector.java:397) at org.eclipse.jetty.server.AbstractConnector$Acceptor.run(AbstractConnector.java:601) at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:671) at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:589) at java.lang.Thread.run(java.base@11.0.14/Thread.java:829) "qtp1552055888-111" #111 daemon prio=5 os_prio=0 cpu=6.48ms elapsed=6576.04s tid=0x00007fd124d6a800 nid=0x8b35 waiting on condition [0x00007fd09a9fe000] java.lang.Thread.State: TIMED_WAITING (parking) at jdk.internal.misc.Unsafe.park(java.base@11.0.14/Native Method) - parking to wait for <0x000000077b21e7d0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(java.base@11.0.14/LockSupport.java:234) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(java.base@11.0.14/AbstractQueuedSynchronizer.java:2123) at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392) at org.eclipse.jetty.util.thread.QueuedThreadPool.idleJobPoll(QueuedThreadPool.java:563) at org.eclipse.jetty.util.thread.QueuedThreadPool.access$800(QueuedThreadPool.java:48) at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:626) at java.lang.Thread.run(java.base@11.0.14/Thread.java:829) "org.eclipse.jetty.server.session.HashSessionManager@17f33ce6Timer" #112 daemon prio=5 os_prio=0 cpu=14.68ms elapsed=6576.04s tid=0x00007fd124d6c800 nid=0x8b36 waiting on condition [0x00007fd09a8fe000] java.lang.Thread.State: TIMED_WAITING (parking) at jdk.internal.misc.Unsafe.park(java.base@11.0.14/Native Method) - parking to wait for <0x000000077b239828> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(java.base@11.0.14/LockSupport.java:234) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(java.base@11.0.14/AbstractQueuedSynchronizer.java:2123) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(java.base@11.0.14/ScheduledThreadPoolExecutor.java:1182) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(java.base@11.0.14/ScheduledThreadPoolExecutor.java:899) at java.util.concurrent.ThreadPoolExecutor.getTask(java.base@11.0.14/ThreadPoolExecutor.java:1054) at java.util.concurrent.ThreadPoolExecutor.runWorker(java.base@11.0.14/ThreadPoolExecutor.java:1114) at java.util.concurrent.ThreadPoolExecutor$Worker.run(java.base@11.0.14/ThreadPoolExecutor.java:628) at java.lang.Thread.run(java.base@11.0.14/Thread.java:829) "org.eclipse.jetty.server.session.HashSessionManager@61ca1957Timer" #113 daemon prio=5 os_prio=0 cpu=12.33ms elapsed=6576.04s tid=0x00007fd124d6e800 nid=0x8b37 waiting on condition [0x00007fd09a7fe000] java.lang.Thread.State: TIMED_WAITING (parking) at jdk.internal.misc.Unsafe.park(java.base@11.0.14/Native Method) - parking to wait for <0x000000077b239910> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(java.base@11.0.14/LockSupport.java:234) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(java.base@11.0.14/AbstractQueuedSynchronizer.java:2123) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(java.base@11.0.14/ScheduledThreadPoolExecutor.java:1182) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(java.base@11.0.14/ScheduledThreadPoolExecutor.java:899) at java.util.concurrent.ThreadPoolExecutor.getTask(java.base@11.0.14/ThreadPoolExecutor.java:1054) at java.util.concurrent.ThreadPoolExecutor.runWorker(java.base@11.0.14/ThreadPoolExecutor.java:1114) at java.util.concurrent.ThreadPoolExecutor$Worker.run(java.base@11.0.14/ThreadPoolExecutor.java:628) at java.lang.Thread.run(java.base@11.0.14/Thread.java:829) "IPC Client (839674462) connection to zhangduo-ubuntu/127.0.0.1:40133 from zhangduo" #114 daemon prio=5 os_prio=0 cpu=1148.67ms elapsed=6575.99s tid=0x00007fd0f004f000 nid=0x8b38 in Object.wait() [0x00007fd09a6fe000] java.lang.Thread.State: TIMED_WAITING (on object monitor) at java.lang.Object.wait(java.base@11.0.14/Native Method) - waiting on at org.apache.hadoop.ipc.Client$Connection.waitForWork(Client.java:1024) - waiting to re-lock in wait() <0x000000077b159460> (a org.apache.hadoop.ipc.Client$Connection) at org.apache.hadoop.ipc.Client$Connection.run(Client.java:1068) "org.eclipse.jetty.server.session.HashSessionManager@67b67a0bTimer" #116 daemon prio=5 os_prio=0 cpu=15.36ms elapsed=6575.95s tid=0x00007fd124c60800 nid=0x8b3a waiting on condition [0x00007fd09a4fe000] java.lang.Thread.State: TIMED_WAITING (parking) at jdk.internal.misc.Unsafe.park(java.base@11.0.14/Native Method) - parking to wait for <0x000000077b239a68> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(java.base@11.0.14/LockSupport.java:234) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(java.base@11.0.14/AbstractQueuedSynchronizer.java:2123) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(java.base@11.0.14/ScheduledThreadPoolExecutor.java:1182) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(java.base@11.0.14/ScheduledThreadPoolExecutor.java:899) at java.util.concurrent.ThreadPoolExecutor.getTask(java.base@11.0.14/ThreadPoolExecutor.java:1054) at java.util.concurrent.ThreadPoolExecutor.runWorker(java.base@11.0.14/ThreadPoolExecutor.java:1114) at java.util.concurrent.ThreadPoolExecutor$Worker.run(java.base@11.0.14/ThreadPoolExecutor.java:628) at java.lang.Thread.run(java.base@11.0.14/Thread.java:829) "nioEventLoopGroup-4-1" #117 prio=10 os_prio=0 cpu=1.35ms elapsed=6575.95s tid=0x00007fd124d7a800 nid=0x8b3b runnable [0x00007fd09a1fe000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPoll.wait(java.base@11.0.14/Native Method) at sun.nio.ch.EPollSelectorImpl.doSelect(java.base@11.0.14/EPollSelectorImpl.java:120) at sun.nio.ch.SelectorImpl.lockAndDoSelect(java.base@11.0.14/SelectorImpl.java:124) - locked <0x000000077b241a10> (a io.netty.channel.nio.SelectedSelectionKeySet) - locked <0x000000077b2419b8> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(java.base@11.0.14/SelectorImpl.java:141) at io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) at io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:803) at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:457) at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:989) at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) at io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) at java.lang.Thread.run(java.base@11.0.14/Thread.java:829) "org.apache.hadoop.util.JvmPauseMonitor$Monitor@249109e" #118 daemon prio=5 os_prio=0 cpu=1262.93ms elapsed=6575.95s tid=0x00007fd124d78800 nid=0x8b3c waiting on condition [0x00007fd09a0fe000] java.lang.Thread.State: TIMED_WAITING (sleeping) at java.lang.Thread.sleep(java.base@11.0.14/Native Method) at org.apache.hadoop.util.JvmPauseMonitor$Monitor.run(JvmPauseMonitor.java:192) at java.lang.Thread.run(java.base@11.0.14/Thread.java:829) "Socket Reader #1 for port 33591" #120 daemon prio=5 os_prio=0 cpu=0.61ms elapsed=6575.95s tid=0x00007fd124d90800 nid=0x8b3d runnable [0x00007fd099ffe000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPoll.wait(java.base@11.0.14/Native Method) at sun.nio.ch.EPollSelectorImpl.doSelect(java.base@11.0.14/EPollSelectorImpl.java:120) at sun.nio.ch.SelectorImpl.lockAndDoSelect(java.base@11.0.14/SelectorImpl.java:124) - locked <0x000000077b241ba0> (a sun.nio.ch.Util$2) - locked <0x000000077b241b48> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(java.base@11.0.14/SelectorImpl.java:141) at org.apache.hadoop.ipc.Server$Listener$Reader.doRunLoop(Server.java:1097) - locked <0x000000077a3eec20> (a org.apache.hadoop.ipc.Server$Listener$Reader) at org.apache.hadoop.ipc.Server$Listener$Reader.run(Server.java:1076) "IPC Server idle connection scanner for port 33591" #121 daemon prio=5 os_prio=0 cpu=36.44ms elapsed=6575.95s tid=0x00007fd124d92800 nid=0x8b3e in Object.wait() [0x00007fd099efe000] java.lang.Thread.State: TIMED_WAITING (on object monitor) at java.lang.Object.wait(java.base@11.0.14/Native Method) - waiting on at java.util.TimerThread.mainLoop(java.base@11.0.14/Timer.java:553) - waiting to re-lock in wait() <0x000000077b241df0> (a java.util.TaskQueue) at java.util.TimerThread.run(java.base@11.0.14/Timer.java:506) "BP-1865003422-127.0.0.1-1652709596755 heartbeating to zhangduo-ubuntu/127.0.0.1:40133" #125 daemon prio=5 os_prio=0 cpu=1662.73ms elapsed=6575.94s tid=0x00007fd124d98000 nid=0x8b3f in Object.wait() [0x00007fd099dfe000] java.lang.Thread.State: TIMED_WAITING (on object monitor) at java.lang.Object.wait(java.base@11.0.14/Native Method) - waiting on at org.apache.hadoop.hdfs.server.datanode.IncrementalBlockReportManager.waitTillNextIBR(IncrementalBlockReportManager.java:158) - waiting to re-lock in wait() <0x000000077b29bae8> (a org.apache.hadoop.hdfs.server.datanode.IncrementalBlockReportManager) at org.apache.hadoop.hdfs.server.datanode.BPServiceActor.offerService(BPServiceActor.java:717) at org.apache.hadoop.hdfs.server.datanode.BPServiceActor.run(BPServiceActor.java:844) at java.lang.Thread.run(java.base@11.0.14/Thread.java:829) "pool-20-thread-1" #126 prio=5 os_prio=0 cpu=362.24ms elapsed=6575.94s tid=0x00007fd124d9a800 nid=0x8b40 waiting on condition [0x00007fd099cfe000] java.lang.Thread.State: TIMED_WAITING (parking) at jdk.internal.misc.Unsafe.park(java.base@11.0.14/Native Method) - parking to wait for <0x000000077b2d1220> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(java.base@11.0.14/LockSupport.java:234) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(java.base@11.0.14/AbstractQueuedSynchronizer.java:2123) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(java.base@11.0.14/ScheduledThreadPoolExecutor.java:1182) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(java.base@11.0.14/ScheduledThreadPoolExecutor.java:899) at java.util.concurrent.ThreadPoolExecutor.getTask(java.base@11.0.14/ThreadPoolExecutor.java:1054) at java.util.concurrent.ThreadPoolExecutor.runWorker(java.base@11.0.14/ThreadPoolExecutor.java:1114) at java.util.concurrent.ThreadPoolExecutor$Worker.run(java.base@11.0.14/ThreadPoolExecutor.java:628) at java.lang.Thread.run(java.base@11.0.14/Thread.java:829) "org.apache.hadoop.hdfs.server.datanode.DataXceiverServer@150b54b2" #107 daemon prio=5 os_prio=0 cpu=1097.07ms elapsed=6575.94s tid=0x00007fd124d9b800 nid=0x8b41 runnable [0x00007fd099bfe000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.ServerSocketChannelImpl.accept0(java.base@11.0.14/Native Method) at sun.nio.ch.ServerSocketChannelImpl.accept(java.base@11.0.14/ServerSocketChannelImpl.java:533) at sun.nio.ch.ServerSocketChannelImpl.accept(java.base@11.0.14/ServerSocketChannelImpl.java:285) at sun.nio.ch.ServerSocketAdaptor.accept(java.base@11.0.14/ServerSocketAdaptor.java:114) - locked <0x000000077b2cd898> (a java.lang.Object) at org.apache.hadoop.hdfs.net.TcpPeerServer.accept(TcpPeerServer.java:85) at org.apache.hadoop.hdfs.server.datanode.DataXceiverServer.run(DataXceiverServer.java:145) at java.lang.Thread.run(java.base@11.0.14/Thread.java:829) "IPC Server Responder" #122 daemon prio=5 os_prio=0 cpu=0.85ms elapsed=6575.94s tid=0x00007fd124d9d000 nid=0x8b42 runnable [0x00007fd099afe000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPoll.wait(java.base@11.0.14/Native Method) at sun.nio.ch.EPollSelectorImpl.doSelect(java.base@11.0.14/EPollSelectorImpl.java:120) at sun.nio.ch.SelectorImpl.lockAndDoSelect(java.base@11.0.14/SelectorImpl.java:124) - locked <0x000000077b2b32e0> (a sun.nio.ch.Util$2) - locked <0x000000077b2b3190> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(java.base@11.0.14/SelectorImpl.java:136) at org.apache.hadoop.ipc.Server$Responder.doRunLoop(Server.java:1334) at org.apache.hadoop.ipc.Server$Responder.run(Server.java:1317) "IPC Server listener on 33591" #119 daemon prio=5 os_prio=0 cpu=0.35ms elapsed=6575.94s tid=0x00007fd124d9f000 nid=0x8b43 runnable [0x00007fd0999fe000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPoll.wait(java.base@11.0.14/Native Method) at sun.nio.ch.EPollSelectorImpl.doSelect(java.base@11.0.14/EPollSelectorImpl.java:120) at sun.nio.ch.SelectorImpl.lockAndDoSelect(java.base@11.0.14/SelectorImpl.java:124) - locked <0x000000077b2b2d88> (a sun.nio.ch.Util$2) - locked <0x000000077b2b2988> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(java.base@11.0.14/SelectorImpl.java:141) at org.apache.hadoop.ipc.Server$Listener.run(Server.java:1159) "IPC Server handler 0 on 33591" #127 daemon prio=5 os_prio=0 cpu=235.01ms elapsed=6575.94s tid=0x00007fd124da0800 nid=0x8b44 waiting on condition [0x00007fd0998fe000] java.lang.Thread.State: TIMED_WAITING (parking) at jdk.internal.misc.Unsafe.park(java.base@11.0.14/Native Method) - parking to wait for <0x000000077b2b18d0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(java.base@11.0.14/LockSupport.java:234) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(java.base@11.0.14/AbstractQueuedSynchronizer.java:2123) at java.util.concurrent.LinkedBlockingQueue.poll(java.base@11.0.14/LinkedBlockingQueue.java:458) at org.apache.hadoop.ipc.CallQueueManager.take(CallQueueManager.java:288) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:2668) "IPC Server handler 1 on 33591" #128 daemon prio=5 os_prio=0 cpu=237.54ms elapsed=6575.94s tid=0x00007fd124da2800 nid=0x8b45 waiting on condition [0x00007fd0997fe000] java.lang.Thread.State: TIMED_WAITING (parking) at jdk.internal.misc.Unsafe.park(java.base@11.0.14/Native Method) - parking to wait for <0x000000077b2b18d0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(java.base@11.0.14/LockSupport.java:234) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(java.base@11.0.14/AbstractQueuedSynchronizer.java:2123) at java.util.concurrent.LinkedBlockingQueue.poll(java.base@11.0.14/LinkedBlockingQueue.java:458) at org.apache.hadoop.ipc.CallQueueManager.take(CallQueueManager.java:288) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:2668) "IPC Server handler 2 on 33591" #129 daemon prio=5 os_prio=0 cpu=242.09ms elapsed=6575.94s tid=0x00007fd124da4800 nid=0x8b46 waiting on condition [0x00007fd0996fe000] java.lang.Thread.State: TIMED_WAITING (parking) at jdk.internal.misc.Unsafe.park(java.base@11.0.14/Native Method) - parking to wait for <0x000000077b2b18d0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(java.base@11.0.14/LockSupport.java:234) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(java.base@11.0.14/AbstractQueuedSynchronizer.java:2123) at java.util.concurrent.LinkedBlockingQueue.poll(java.base@11.0.14/LinkedBlockingQueue.java:458) at org.apache.hadoop.ipc.CallQueueManager.take(CallQueueManager.java:288) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:2668) "IPC Server handler 3 on 33591" #130 daemon prio=5 os_prio=0 cpu=241.24ms elapsed=6575.94s tid=0x00007fd124da6000 nid=0x8b47 waiting on condition [0x00007fd0995fe000] java.lang.Thread.State: TIMED_WAITING (parking) at jdk.internal.misc.Unsafe.park(java.base@11.0.14/Native Method) - parking to wait for <0x000000077b2b18d0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(java.base@11.0.14/LockSupport.java:234) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(java.base@11.0.14/AbstractQueuedSynchronizer.java:2123) at java.util.concurrent.LinkedBlockingQueue.poll(java.base@11.0.14/LinkedBlockingQueue.java:458) at org.apache.hadoop.ipc.CallQueueManager.take(CallQueueManager.java:288) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:2668) "IPC Server handler 4 on 33591" #131 daemon prio=5 os_prio=0 cpu=232.65ms elapsed=6575.94s tid=0x00007fd124da8000 nid=0x8b48 waiting on condition [0x00007fd0994fe000] java.lang.Thread.State: TIMED_WAITING (parking) at jdk.internal.misc.Unsafe.park(java.base@11.0.14/Native Method) - parking to wait for <0x000000077b2b18d0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(java.base@11.0.14/LockSupport.java:234) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(java.base@11.0.14/AbstractQueuedSynchronizer.java:2123) at java.util.concurrent.LinkedBlockingQueue.poll(java.base@11.0.14/LinkedBlockingQueue.java:458) at org.apache.hadoop.ipc.CallQueueManager.take(CallQueueManager.java:288) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:2668) "VolumeScannerThread(/home/zhangduo/hbase/code/hbase-server/target/test-data/6526c0c5-3c73-030e-181d-98a6d22d8b51/cluster_23b7b4bd-3db7-c032-0531-3d35240dca27/dfs/data/data1)" #136 daemon prio=5 os_prio=0 cpu=6.07ms elapsed=6575.76s tid=0x00007fd0f0096800 nid=0x8b51 in Object.wait() [0x00007fd0992fe000] java.lang.Thread.State: TIMED_WAITING (on object monitor) at java.lang.Object.wait(java.base@11.0.14/Native Method) - waiting on <0x000000077b2066d0> (a org.apache.hadoop.hdfs.server.datanode.VolumeScanner) at org.apache.hadoop.hdfs.server.datanode.VolumeScanner.run(VolumeScanner.java:626) - waiting to re-lock in wait() <0x000000077b2066d0> (a org.apache.hadoop.hdfs.server.datanode.VolumeScanner) "VolumeScannerThread(/home/zhangduo/hbase/code/hbase-server/target/test-data/6526c0c5-3c73-030e-181d-98a6d22d8b51/cluster_23b7b4bd-3db7-c032-0531-3d35240dca27/dfs/data/data3)" #137 daemon prio=5 os_prio=0 cpu=2.27ms elapsed=6575.76s tid=0x00007fd1607b2000 nid=0x8b52 in Object.wait() [0x00007fd0993fe000] java.lang.Thread.State: TIMED_WAITING (on object monitor) at java.lang.Object.wait(java.base@11.0.14/Native Method) - waiting on <0x000000077b2a7138> (a org.apache.hadoop.hdfs.server.datanode.VolumeScanner) at org.apache.hadoop.hdfs.server.datanode.VolumeScanner.run(VolumeScanner.java:626) - waiting to re-lock in wait() <0x000000077b2a7138> (a org.apache.hadoop.hdfs.server.datanode.VolumeScanner) "VolumeScannerThread(/home/zhangduo/hbase/code/hbase-server/target/test-data/6526c0c5-3c73-030e-181d-98a6d22d8b51/cluster_23b7b4bd-3db7-c032-0531-3d35240dca27/dfs/data/data2)" #138 daemon prio=5 os_prio=0 cpu=3.55ms elapsed=6575.76s tid=0x00007fd0f0097800 nid=0x8b53 in Object.wait() [0x00007fd098dfe000] java.lang.Thread.State: TIMED_WAITING (on object monitor) at java.lang.Object.wait(java.base@11.0.14/Native Method) - waiting on <0x000000077b20af40> (a org.apache.hadoop.hdfs.server.datanode.VolumeScanner) at org.apache.hadoop.hdfs.server.datanode.VolumeScanner.run(VolumeScanner.java:626) - waiting to re-lock in wait() <0x000000077b20af40> (a org.apache.hadoop.hdfs.server.datanode.VolumeScanner) "VolumeScannerThread(/home/zhangduo/hbase/code/hbase-server/target/test-data/6526c0c5-3c73-030e-181d-98a6d22d8b51/cluster_23b7b4bd-3db7-c032-0531-3d35240dca27/dfs/data/data4)" #139 daemon prio=5 os_prio=0 cpu=6.02ms elapsed=6575.75s tid=0x00007fd1607b3000 nid=0x8b54 in Object.wait() [0x00007fd098cfe000] java.lang.Thread.State: TIMED_WAITING (on object monitor) at java.lang.Object.wait(java.base@11.0.14/Native Method) - waiting on <0x000000077b2c91e0> (a org.apache.hadoop.hdfs.server.datanode.VolumeScanner) at org.apache.hadoop.hdfs.server.datanode.VolumeScanner.run(VolumeScanner.java:626) - waiting to re-lock in wait() <0x000000077b2c91e0> (a org.apache.hadoop.hdfs.server.datanode.VolumeScanner) "pool-17-thread-1" #142 daemon prio=5 os_prio=0 cpu=0.28ms elapsed=6575.74s tid=0x00007fd1607b7800 nid=0x8b57 waiting on condition [0x00007fd0989fe000] java.lang.Thread.State: WAITING (parking) at jdk.internal.misc.Unsafe.park(java.base@11.0.14/Native Method) - parking to wait for <0x000000077b2d0a10> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.park(java.base@11.0.14/LockSupport.java:194) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(java.base@11.0.14/AbstractQueuedSynchronizer.java:2081) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(java.base@11.0.14/ScheduledThreadPoolExecutor.java:1170) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(java.base@11.0.14/ScheduledThreadPoolExecutor.java:899) at java.util.concurrent.ThreadPoolExecutor.getTask(java.base@11.0.14/ThreadPoolExecutor.java:1054) at java.util.concurrent.ThreadPoolExecutor.runWorker(java.base@11.0.14/ThreadPoolExecutor.java:1114) at java.util.concurrent.ThreadPoolExecutor$Worker.run(java.base@11.0.14/ThreadPoolExecutor.java:628) at java.lang.Thread.run(java.base@11.0.14/Thread.java:829) "pool-10-thread-1" #143 daemon prio=5 os_prio=0 cpu=0.31ms elapsed=6575.74s tid=0x00007fd0f00a5000 nid=0x8b58 waiting on condition [0x00007fd0988fe000] java.lang.Thread.State: WAITING (parking) at jdk.internal.misc.Unsafe.park(java.base@11.0.14/Native Method) - parking to wait for <0x000000077b2179a8> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.park(java.base@11.0.14/LockSupport.java:194) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(java.base@11.0.14/AbstractQueuedSynchronizer.java:2081) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(java.base@11.0.14/ScheduledThreadPoolExecutor.java:1170) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(java.base@11.0.14/ScheduledThreadPoolExecutor.java:899) at java.util.concurrent.ThreadPoolExecutor.getTask(java.base@11.0.14/ThreadPoolExecutor.java:1054) at java.util.concurrent.ThreadPoolExecutor.runWorker(java.base@11.0.14/ThreadPoolExecutor.java:1114) at java.util.concurrent.ThreadPoolExecutor$Worker.run(java.base@11.0.14/ThreadPoolExecutor.java:628) at java.lang.Thread.run(java.base@11.0.14/Thread.java:829) "refreshUsed-/home/zhangduo/hbase/code/hbase-server/target/test-data/6526c0c5-3c73-030e-181d-98a6d22d8b51/cluster_23b7b4bd-3db7-c032-0531-3d35240dca27/dfs/data/data1/current/BP-1865003422-127.0.0.1-1652709596755" #154 daemon prio=5 os_prio=0 cpu=9.74ms elapsed=6575.71s tid=0x00007fd0b405b800 nid=0x8b67 waiting on condition [0x00007fd0980fe000] java.lang.Thread.State: TIMED_WAITING (sleeping) at java.lang.Thread.sleep(java.base@11.0.14/Native Method) at org.apache.hadoop.fs.CachingGetSpaceUsed$RefreshThread.run(CachingGetSpaceUsed.java:179) at java.lang.Thread.run(java.base@11.0.14/Thread.java:829) "refreshUsed-/home/zhangduo/hbase/code/hbase-server/target/test-data/6526c0c5-3c73-030e-181d-98a6d22d8b51/cluster_23b7b4bd-3db7-c032-0531-3d35240dca27/dfs/data/data4/current/BP-1865003422-127.0.0.1-1652709596755" #156 daemon prio=5 os_prio=0 cpu=11.24ms elapsed=6575.71s tid=0x00007fd0c800f000 nid=0x8b68 waiting on condition [0x00007fd0981fe000] java.lang.Thread.State: TIMED_WAITING (sleeping) at java.lang.Thread.sleep(java.base@11.0.14/Native Method) at org.apache.hadoop.fs.CachingGetSpaceUsed$RefreshThread.run(CachingGetSpaceUsed.java:179) at java.lang.Thread.run(java.base@11.0.14/Thread.java:829) "refreshUsed-/home/zhangduo/hbase/code/hbase-server/target/test-data/6526c0c5-3c73-030e-181d-98a6d22d8b51/cluster_23b7b4bd-3db7-c032-0531-3d35240dca27/dfs/data/data3/current/BP-1865003422-127.0.0.1-1652709596755" #155 daemon prio=5 os_prio=0 cpu=13.14ms elapsed=6575.71s tid=0x00007fd0c0040800 nid=0x8b69 waiting on condition [0x00007fd0983fe000] java.lang.Thread.State: TIMED_WAITING (sleeping) at java.lang.Thread.sleep(java.base@11.0.14/Native Method) at org.apache.hadoop.fs.CachingGetSpaceUsed$RefreshThread.run(CachingGetSpaceUsed.java:179) at java.lang.Thread.run(java.base@11.0.14/Thread.java:829) "refreshUsed-/home/zhangduo/hbase/code/hbase-server/target/test-data/6526c0c5-3c73-030e-181d-98a6d22d8b51/cluster_23b7b4bd-3db7-c032-0531-3d35240dca27/dfs/data/data2/current/BP-1865003422-127.0.0.1-1652709596755" #157 daemon prio=5 os_prio=0 cpu=9.58ms elapsed=6575.71s tid=0x00007fd0bc003000 nid=0x8b6a waiting on condition [0x00007fd0982fe000] java.lang.Thread.State: TIMED_WAITING (sleeping) at java.lang.Thread.sleep(java.base@11.0.14/Native Method) at org.apache.hadoop.fs.CachingGetSpaceUsed$RefreshThread.run(CachingGetSpaceUsed.java:179) at java.lang.Thread.run(java.base@11.0.14/Thread.java:829) "ForkJoinPool-2-worker-5" #161 daemon prio=5 os_prio=0 cpu=0.51ms elapsed=6575.70s tid=0x00007fd0f8002000 nid=0x8b6e waiting on condition [0x00007fd093ffe000] java.lang.Thread.State: WAITING (parking) at jdk.internal.misc.Unsafe.park(java.base@11.0.14/Native Method) - parking to wait for <0x000000077b2d5f28> (a java.util.concurrent.ForkJoinPool) at java.util.concurrent.locks.LockSupport.park(java.base@11.0.14/LockSupport.java:194) at java.util.concurrent.ForkJoinPool.runWorker(java.base@11.0.14/ForkJoinPool.java:1628) at java.util.concurrent.ForkJoinWorkerThread.run(java.base@11.0.14/ForkJoinWorkerThread.java:183) "java.util.concurrent.ThreadPoolExecutor$Worker@2bae0ab8[State = -1, empty queue]" #172 daemon prio=5 os_prio=0 cpu=0.16ms elapsed=6575.67s tid=0x00007fd1607a0000 nid=0x8b75 waiting on condition [0x00007fd093afe000] java.lang.Thread.State: TIMED_WAITING (parking) at jdk.internal.misc.Unsafe.park(java.base@11.0.14/Native Method) - parking to wait for <0x000000077b2cef40> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(java.base@11.0.14/LockSupport.java:234) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(java.base@11.0.14/AbstractQueuedSynchronizer.java:2123) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(java.base@11.0.14/ScheduledThreadPoolExecutor.java:1182) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(java.base@11.0.14/ScheduledThreadPoolExecutor.java:899) at java.util.concurrent.ThreadPoolExecutor.getTask(java.base@11.0.14/ThreadPoolExecutor.java:1054) at java.util.concurrent.ThreadPoolExecutor.runWorker(java.base@11.0.14/ThreadPoolExecutor.java:1114) at java.util.concurrent.ThreadPoolExecutor$Worker.run(java.base@11.0.14/ThreadPoolExecutor.java:628) at java.lang.Thread.run(java.base@11.0.14/Thread.java:829) "java.util.concurrent.ThreadPoolExecutor$Worker@597ef57c[State = -1, empty queue]" #173 daemon prio=5 os_prio=0 cpu=69.13ms elapsed=6575.67s tid=0x00007fd0f0062000 nid=0x8b76 waiting on condition [0x00007fd0985fe000] java.lang.Thread.State: TIMED_WAITING (parking) at jdk.internal.misc.Unsafe.park(java.base@11.0.14/Native Method) - parking to wait for <0x000000077b2114f0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(java.base@11.0.14/LockSupport.java:234) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(java.base@11.0.14/AbstractQueuedSynchronizer.java:2123) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(java.base@11.0.14/ScheduledThreadPoolExecutor.java:1182) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(java.base@11.0.14/ScheduledThreadPoolExecutor.java:899) at java.util.concurrent.ThreadPoolExecutor.getTask(java.base@11.0.14/ThreadPoolExecutor.java:1054) at java.util.concurrent.ThreadPoolExecutor.runWorker(java.base@11.0.14/ThreadPoolExecutor.java:1114) at java.util.concurrent.ThreadPoolExecutor$Worker.run(java.base@11.0.14/ThreadPoolExecutor.java:628) at java.lang.Thread.run(java.base@11.0.14/Thread.java:829) "IPC Client (839674462) connection to zhangduo-ubuntu/127.0.0.1:40133 from zhangduo" #175 daemon prio=5 os_prio=0 cpu=2213766.13ms elapsed=6575.58s tid=0x00007fd124ed3000 nid=0x8b77 runnable [0x00007fd093bfe000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPoll.wait(java.base@11.0.14/Native Method) at sun.nio.ch.EPollSelectorImpl.doSelect(java.base@11.0.14/EPollSelectorImpl.java:120) at sun.nio.ch.SelectorImpl.lockAndDoSelect(java.base@11.0.14/SelectorImpl.java:124) - locked <0x000000078203ee00> (a sun.nio.ch.Util$2) - locked <0x000000078203eba0> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(java.base@11.0.14/SelectorImpl.java:136) at org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) at org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:157) at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) at java.io.FilterInputStream.read(java.base@11.0.14/FilterInputStream.java:133) at java.io.BufferedInputStream.fill(java.base@11.0.14/BufferedInputStream.java:252) at java.io.BufferedInputStream.read(java.base@11.0.14/BufferedInputStream.java:271) - locked <0x000000077c1072d0> (a java.io.BufferedInputStream) at java.io.FilterInputStream.read(java.base@11.0.14/FilterInputStream.java:83) at java.io.FilterInputStream.read(java.base@11.0.14/FilterInputStream.java:83) at org.apache.hadoop.ipc.Client$Connection$PingInputStream.read(Client.java:557) at java.io.DataInputStream.readInt(java.base@11.0.14/DataInputStream.java:392) at org.apache.hadoop.ipc.Client$IpcStreams.readResponse(Client.java:1816) at org.apache.hadoop.ipc.Client$Connection.receiveRpcResponse(Client.java:1173) at org.apache.hadoop.ipc.Client$Connection.run(Client.java:1069) "NIOServerCxnFactory.SelectorThread-1" #179 daemon prio=5 os_prio=0 cpu=95.96ms elapsed=6575.49s tid=0x00007fd124fac800 nid=0x8b7b runnable [0x00007fd0986fe000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPoll.wait(java.base@11.0.14/Native Method) at sun.nio.ch.EPollSelectorImpl.doSelect(java.base@11.0.14/EPollSelectorImpl.java:120) at sun.nio.ch.SelectorImpl.lockAndDoSelect(java.base@11.0.14/SelectorImpl.java:124) - locked <0x000000077b7cb838> (a sun.nio.ch.Util$2) - locked <0x000000077b7cb7e0> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(java.base@11.0.14/SelectorImpl.java:141) at org.apache.zookeeper.server.NIOServerCnxnFactory$SelectorThread.select(NIOServerCnxnFactory.java:426) at org.apache.zookeeper.server.NIOServerCnxnFactory$SelectorThread.run(NIOServerCnxnFactory.java:391) "NIOServerCxnFactory.SelectorThread-0" #178 daemon prio=5 os_prio=0 cpu=302.83ms elapsed=6575.49s tid=0x00007fd124fad800 nid=0x8b7c runnable [0x00007fd0935fe000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPoll.wait(java.base@11.0.14/Native Method) at sun.nio.ch.EPollSelectorImpl.doSelect(java.base@11.0.14/EPollSelectorImpl.java:120) at sun.nio.ch.SelectorImpl.lockAndDoSelect(java.base@11.0.14/SelectorImpl.java:124) - locked <0x000000077b75cbf0> (a sun.nio.ch.Util$2) - locked <0x000000077b75cb98> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(java.base@11.0.14/SelectorImpl.java:141) at org.apache.zookeeper.server.NIOServerCnxnFactory$SelectorThread.select(NIOServerCnxnFactory.java:426) at org.apache.zookeeper.server.NIOServerCnxnFactory$SelectorThread.run(NIOServerCnxnFactory.java:391) "NIOServerCxnFactory.AcceptThread:localhost/127.0.0.1:59087" #180 daemon prio=5 os_prio=0 cpu=6.35ms elapsed=6575.49s tid=0x00007fd124fdd000 nid=0x8b7d runnable [0x00007fd0934fe000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPoll.wait(java.base@11.0.14/Native Method) at sun.nio.ch.EPollSelectorImpl.doSelect(java.base@11.0.14/EPollSelectorImpl.java:120) at sun.nio.ch.SelectorImpl.lockAndDoSelect(java.base@11.0.14/SelectorImpl.java:124) - locked <0x000000077b7f07b0> (a sun.nio.ch.Util$2) - locked <0x000000077b7f0758> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(java.base@11.0.14/SelectorImpl.java:141) at org.apache.zookeeper.server.NIOServerCnxnFactory$AcceptThread.select(NIOServerCnxnFactory.java:229) at org.apache.zookeeper.server.NIOServerCnxnFactory$AcceptThread.run(NIOServerCnxnFactory.java:205) "ConnnectionExpirer" #177 daemon prio=5 os_prio=0 cpu=29.19ms elapsed=6575.49s tid=0x00007fd12502a000 nid=0x8b7e waiting on condition [0x00007fd0933fe000] java.lang.Thread.State: TIMED_WAITING (sleeping) at java.lang.Thread.sleep(java.base@11.0.14/Native Method) at org.apache.zookeeper.server.NIOServerCnxnFactory$ConnectionExpirerThread.run(NIOServerCnxnFactory.java:574) "SessionTracker" #181 daemon prio=5 os_prio=0 cpu=151.73ms elapsed=6575.48s tid=0x00007fd125032000 nid=0x8b7f waiting on condition [0x00007fd0932fe000] java.lang.Thread.State: TIMED_WAITING (sleeping) at java.lang.Thread.sleep(java.base@11.0.14/Native Method) at org.apache.zookeeper.server.SessionTrackerImpl.run(SessionTrackerImpl.java:151) "SyncThread:0" #182 daemon prio=5 os_prio=0 cpu=221.40ms elapsed=6575.47s tid=0x00007fd125035000 nid=0x8b80 waiting on condition [0x00007fd0931fe000] java.lang.Thread.State: WAITING (parking) at jdk.internal.misc.Unsafe.park(java.base@11.0.14/Native Method) - parking to wait for <0x000000077b7a6e20> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.park(java.base@11.0.14/LockSupport.java:194) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(java.base@11.0.14/AbstractQueuedSynchronizer.java:2081) at java.util.concurrent.LinkedBlockingQueue.take(java.base@11.0.14/LinkedBlockingQueue.java:433) at org.apache.zookeeper.server.SyncRequestProcessor.run(SyncRequestProcessor.java:109) "ProcessThread(sid:0 cport:59087):" #183 daemon prio=5 os_prio=0 cpu=125.22ms elapsed=6575.47s tid=0x00007fd12503a800 nid=0x8b81 waiting on condition [0x00007fd0930fe000] java.lang.Thread.State: WAITING (parking) at jdk.internal.misc.Unsafe.park(java.base@11.0.14/Native Method) - parking to wait for <0x000000077b75ce58> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.park(java.base@11.0.14/LockSupport.java:194) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(java.base@11.0.14/AbstractQueuedSynchronizer.java:2081) at java.util.concurrent.LinkedBlockingQueue.take(java.base@11.0.14/LinkedBlockingQueue.java:433) at org.apache.zookeeper.server.PrepRequestProcessor.run(PrepRequestProcessor.java:134) "NIOWorkerThread-1" #184 daemon prio=5 os_prio=0 cpu=31.81ms elapsed=6575.45s tid=0x00007fd1280a3000 nid=0x8b82 waiting on condition [0x00007fd092ffe000] java.lang.Thread.State: WAITING (parking) at jdk.internal.misc.Unsafe.park(java.base@11.0.14/Native Method) - parking to wait for <0x000000077b7cb848> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.park(java.base@11.0.14/LockSupport.java:194) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(java.base@11.0.14/AbstractQueuedSynchronizer.java:2081) at java.util.concurrent.LinkedBlockingQueue.take(java.base@11.0.14/LinkedBlockingQueue.java:433) at java.util.concurrent.ThreadPoolExecutor.getTask(java.base@11.0.14/ThreadPoolExecutor.java:1054) at java.util.concurrent.ThreadPoolExecutor.runWorker(java.base@11.0.14/ThreadPoolExecutor.java:1114) at java.util.concurrent.ThreadPoolExecutor$Worker.run(java.base@11.0.14/ThreadPoolExecutor.java:628) at java.lang.Thread.run(java.base@11.0.14/Thread.java:829) "LeaseRenewer:zhangduo@zhangduo-ubuntu:40133" #186 daemon prio=5 os_prio=0 cpu=304.93ms elapsed=6575.36s tid=0x00007fd125068800 nid=0x8b84 waiting on condition [0x00007fd092dfe000] java.lang.Thread.State: TIMED_WAITING (sleeping) at java.lang.Thread.sleep(java.base@11.0.14/Native Method) at org.apache.hadoop.hdfs.client.impl.LeaseRenewer.run(LeaseRenewer.java:411) at org.apache.hadoop.hdfs.client.impl.LeaseRenewer.access$600(LeaseRenewer.java:76) at org.apache.hadoop.hdfs.client.impl.LeaseRenewer$1.run(LeaseRenewer.java:307) at java.lang.Thread.run(java.base@11.0.14/Thread.java:829) "HBase-Metrics2-1" #193 daemon prio=5 os_prio=0 cpu=1004.12ms elapsed=6574.66s tid=0x00007fd125364800 nid=0x8b8b waiting on condition [0x00007fd11c910000] java.lang.Thread.State: TIMED_WAITING (parking) at jdk.internal.misc.Unsafe.park(java.base@11.0.14/Native Method) - parking to wait for <0x000000077b75cf38> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(java.base@11.0.14/LockSupport.java:234) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(java.base@11.0.14/AbstractQueuedSynchronizer.java:2123) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(java.base@11.0.14/ScheduledThreadPoolExecutor.java:1182) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(java.base@11.0.14/ScheduledThreadPoolExecutor.java:899) at java.util.concurrent.ThreadPoolExecutor.getTask(java.base@11.0.14/ThreadPoolExecutor.java:1054) at java.util.concurrent.ThreadPoolExecutor.runWorker(java.base@11.0.14/ThreadPoolExecutor.java:1114) at java.util.concurrent.ThreadPoolExecutor$Worker.run(java.base@11.0.14/ThreadPoolExecutor.java:628) at java.lang.Thread.run(java.base@11.0.14/Thread.java:829) "RS-EventLoopGroup-1-1" #194 daemon prio=10 os_prio=0 cpu=41.33ms elapsed=6574.16s tid=0x00007fd124818000 nid=0x8b90 runnable [0x00007fd11c310000] java.lang.Thread.State: RUNNABLE at org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) at org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:193) at org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:186) at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:290) at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:347) at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:986) at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) at java.lang.Thread.run(java.base@11.0.14/Thread.java:829) "Time-limited test-SendThread(127.0.0.1:59087)" #195 daemon prio=5 os_prio=0 cpu=144.27ms elapsed=6574.12s tid=0x00007fd12500a000 nid=0x8b91 runnable [0x00007fd092afe000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPoll.wait(java.base@11.0.14/Native Method) at sun.nio.ch.EPollSelectorImpl.doSelect(java.base@11.0.14/EPollSelectorImpl.java:120) at sun.nio.ch.SelectorImpl.lockAndDoSelect(java.base@11.0.14/SelectorImpl.java:124) - locked <0x000000077b75d6b0> (a sun.nio.ch.Util$2) - locked <0x000000077b75d198> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(java.base@11.0.14/SelectorImpl.java:136) at org.apache.zookeeper.ClientCnxnSocketNIO.doTransport(ClientCnxnSocketNIO.java:345) at org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1223) "Time-limited test-EventThread" #196 daemon prio=5 os_prio=0 cpu=7.00ms elapsed=6574.12s tid=0x00007fd12500b800 nid=0x8b92 waiting on condition [0x00007fd0925fe000] java.lang.Thread.State: WAITING (parking) at jdk.internal.misc.Unsafe.park(java.base@11.0.14/Native Method) - parking to wait for <0x000000077b7f0b08> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.park(java.base@11.0.14/LockSupport.java:194) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(java.base@11.0.14/AbstractQueuedSynchronizer.java:2081) at java.util.concurrent.LinkedBlockingQueue.take(java.base@11.0.14/LinkedBlockingQueue.java:433) at org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:506) "NIOWorkerThread-2" #197 daemon prio=5 os_prio=0 cpu=27.18ms elapsed=6574.11s tid=0x00007fd134004800 nid=0x8b93 waiting on condition [0x00007fd0924fe000] java.lang.Thread.State: WAITING (parking) at jdk.internal.misc.Unsafe.park(java.base@11.0.14/Native Method) - parking to wait for <0x000000077b7cb848> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.park(java.base@11.0.14/LockSupport.java:194) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(java.base@11.0.14/AbstractQueuedSynchronizer.java:2081) at java.util.concurrent.LinkedBlockingQueue.take(java.base@11.0.14/LinkedBlockingQueue.java:433) at java.util.concurrent.ThreadPoolExecutor.getTask(java.base@11.0.14/ThreadPoolExecutor.java:1054) at java.util.concurrent.ThreadPoolExecutor.runWorker(java.base@11.0.14/ThreadPoolExecutor.java:1114) at java.util.concurrent.ThreadPoolExecutor$Worker.run(java.base@11.0.14/ThreadPoolExecutor.java:628) at java.lang.Thread.run(java.base@11.0.14/Thread.java:829) "NIOWorkerThread-3" #198 daemon prio=5 os_prio=0 cpu=25.27ms elapsed=6574.10s tid=0x00007fd134005800 nid=0x8b94 waiting on condition [0x00007fd0927fe000] java.lang.Thread.State: WAITING (parking) at jdk.internal.misc.Unsafe.park(java.base@11.0.14/Native Method) - parking to wait for <0x000000077b7cb848> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.park(java.base@11.0.14/LockSupport.java:194) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(java.base@11.0.14/AbstractQueuedSynchronizer.java:2081) at java.util.concurrent.LinkedBlockingQueue.take(java.base@11.0.14/LinkedBlockingQueue.java:433) at java.util.concurrent.ThreadPoolExecutor.getTask(java.base@11.0.14/ThreadPoolExecutor.java:1054) at java.util.concurrent.ThreadPoolExecutor.runWorker(java.base@11.0.14/ThreadPoolExecutor.java:1114) at java.util.concurrent.ThreadPoolExecutor$Worker.run(java.base@11.0.14/ThreadPoolExecutor.java:628) at java.lang.Thread.run(java.base@11.0.14/Thread.java:829) "zk-event-processor-pool-0" #199 daemon prio=5 os_prio=0 cpu=8.20ms elapsed=6574.09s tid=0x00007fd184015000 nid=0x8b95 waiting on condition [0x00007fd0926fe000] java.lang.Thread.State: WAITING (parking) at jdk.internal.misc.Unsafe.park(java.base@11.0.14/Native Method) - parking to wait for <0x000000077b7f0c40> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.park(java.base@11.0.14/LockSupport.java:194) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(java.base@11.0.14/AbstractQueuedSynchronizer.java:2081) at java.util.concurrent.LinkedBlockingQueue.take(java.base@11.0.14/LinkedBlockingQueue.java:433) at java.util.concurrent.ThreadPoolExecutor.getTask(java.base@11.0.14/ThreadPoolExecutor.java:1054) at java.util.concurrent.ThreadPoolExecutor.runWorker(java.base@11.0.14/ThreadPoolExecutor.java:1114) at java.util.concurrent.ThreadPoolExecutor$Worker.run(java.base@11.0.14/ThreadPoolExecutor.java:628) at java.lang.Thread.run(java.base@11.0.14/Thread.java:829) "NIOWorkerThread-4" #200 daemon prio=5 os_prio=0 cpu=23.86ms elapsed=6574.09s tid=0x00007fd134007000 nid=0x8b96 waiting on condition [0x00007fd091ffe000] java.lang.Thread.State: WAITING (parking) at jdk.internal.misc.Unsafe.park(java.base@11.0.14/Native Method) - parking to wait for <0x000000077b7cb848> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.park(java.base@11.0.14/LockSupport.java:194) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(java.base@11.0.14/AbstractQueuedSynchronizer.java:2081) at java.util.concurrent.LinkedBlockingQueue.take(java.base@11.0.14/LinkedBlockingQueue.java:433) at java.util.concurrent.ThreadPoolExecutor.getTask(java.base@11.0.14/ThreadPoolExecutor.java:1054) at java.util.concurrent.ThreadPoolExecutor.runWorker(java.base@11.0.14/ThreadPoolExecutor.java:1114) at java.util.concurrent.ThreadPoolExecutor$Worker.run(java.base@11.0.14/ThreadPoolExecutor.java:628) at java.lang.Thread.run(java.base@11.0.14/Thread.java:829) "NIOWorkerThread-5" #201 daemon prio=5 os_prio=0 cpu=24.53ms elapsed=6574.08s tid=0x00007fd134008800 nid=0x8b97 waiting on condition [0x00007fd0911fe000] java.lang.Thread.State: WAITING (parking) at jdk.internal.misc.Unsafe.park(java.base@11.0.14/Native Method) - parking to wait for <0x000000077b7cb848> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.park(java.base@11.0.14/LockSupport.java:194) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(java.base@11.0.14/AbstractQueuedSynchronizer.java:2081) at java.util.concurrent.LinkedBlockingQueue.take(java.base@11.0.14/LinkedBlockingQueue.java:433) at java.util.concurrent.ThreadPoolExecutor.getTask(java.base@11.0.14/ThreadPoolExecutor.java:1054) at java.util.concurrent.ThreadPoolExecutor.runWorker(java.base@11.0.14/ThreadPoolExecutor.java:1114) at java.util.concurrent.ThreadPoolExecutor$Worker.run(java.base@11.0.14/ThreadPoolExecutor.java:628) at java.lang.Thread.run(java.base@11.0.14/Thread.java:829) "NIOWorkerThread-6" #202 daemon prio=5 os_prio=0 cpu=23.80ms elapsed=6574.08s tid=0x00007fd13400a800 nid=0x8b98 waiting on condition [0x00007fd0910fe000] java.lang.Thread.State: WAITING (parking) at jdk.internal.misc.Unsafe.park(java.base@11.0.14/Native Method) - parking to wait for <0x000000077b7cb848> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.park(java.base@11.0.14/LockSupport.java:194) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(java.base@11.0.14/AbstractQueuedSynchronizer.java:2081) at java.util.concurrent.LinkedBlockingQueue.take(java.base@11.0.14/LinkedBlockingQueue.java:433) at java.util.concurrent.ThreadPoolExecutor.getTask(java.base@11.0.14/ThreadPoolExecutor.java:1054) at java.util.concurrent.ThreadPoolExecutor.runWorker(java.base@11.0.14/ThreadPoolExecutor.java:1114) at java.util.concurrent.ThreadPoolExecutor$Worker.run(java.base@11.0.14/ThreadPoolExecutor.java:628) at java.lang.Thread.run(java.base@11.0.14/Thread.java:829) "NIOWorkerThread-7" #203 daemon prio=5 os_prio=0 cpu=23.17ms elapsed=6574.08s tid=0x00007fd13400e800 nid=0x8b99 waiting on condition [0x00007fd090ffe000] java.lang.Thread.State: WAITING (parking) at jdk.internal.misc.Unsafe.park(java.base@11.0.14/Native Method) - parking to wait for <0x000000077b7cb848> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.park(java.base@11.0.14/LockSupport.java:194) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(java.base@11.0.14/AbstractQueuedSynchronizer.java:2081) at java.util.concurrent.LinkedBlockingQueue.take(java.base@11.0.14/LinkedBlockingQueue.java:433) at java.util.concurrent.ThreadPoolExecutor.getTask(java.base@11.0.14/ThreadPoolExecutor.java:1054) at java.util.concurrent.ThreadPoolExecutor.runWorker(java.base@11.0.14/ThreadPoolExecutor.java:1114) at java.util.concurrent.ThreadPoolExecutor$Worker.run(java.base@11.0.14/ThreadPoolExecutor.java:628) at java.lang.Thread.run(java.base@11.0.14/Thread.java:829) "NIOWorkerThread-8" #204 daemon prio=5 os_prio=0 cpu=24.15ms elapsed=6574.07s tid=0x00007fd134010000 nid=0x8b9a waiting on condition [0x00007fd090efe000] java.lang.Thread.State: WAITING (parking) at jdk.internal.misc.Unsafe.park(java.base@11.0.14/Native Method) - parking to wait for <0x000000077b7cb848> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.park(java.base@11.0.14/LockSupport.java:194) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(java.base@11.0.14/AbstractQueuedSynchronizer.java:2081) at java.util.concurrent.LinkedBlockingQueue.take(java.base@11.0.14/LinkedBlockingQueue.java:433) at java.util.concurrent.ThreadPoolExecutor.getTask(java.base@11.0.14/ThreadPoolExecutor.java:1054) at java.util.concurrent.ThreadPoolExecutor.runWorker(java.base@11.0.14/ThreadPoolExecutor.java:1114) at java.util.concurrent.ThreadPoolExecutor$Worker.run(java.base@11.0.14/ThreadPoolExecutor.java:628) at java.lang.Thread.run(java.base@11.0.14/Thread.java:829) "NIOWorkerThread-9" #205 daemon prio=5 os_prio=0 cpu=22.53ms elapsed=6574.07s tid=0x00007fd134012000 nid=0x8b9b waiting on condition [0x00007fd090dfe000] java.lang.Thread.State: WAITING (parking) at jdk.internal.misc.Unsafe.park(java.base@11.0.14/Native Method) - parking to wait for <0x000000077b7cb848> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.park(java.base@11.0.14/LockSupport.java:194) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(java.base@11.0.14/AbstractQueuedSynchronizer.java:2081) at java.util.concurrent.LinkedBlockingQueue.take(java.base@11.0.14/LinkedBlockingQueue.java:433) at java.util.concurrent.ThreadPoolExecutor.getTask(java.base@11.0.14/ThreadPoolExecutor.java:1054) at java.util.concurrent.ThreadPoolExecutor.runWorker(java.base@11.0.14/ThreadPoolExecutor.java:1114) at java.util.concurrent.ThreadPoolExecutor$Worker.run(java.base@11.0.14/ThreadPoolExecutor.java:628) at java.lang.Thread.run(java.base@11.0.14/Thread.java:829) "NIOWorkerThread-10" #206 daemon prio=5 os_prio=0 cpu=22.38ms elapsed=6574.07s tid=0x00007fd134014000 nid=0x8b9c waiting on condition [0x00007fd090cfe000] java.lang.Thread.State: WAITING (parking) at jdk.internal.misc.Unsafe.park(java.base@11.0.14/Native Method) - parking to wait for <0x000000077b7cb848> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.park(java.base@11.0.14/LockSupport.java:194) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(java.base@11.0.14/AbstractQueuedSynchronizer.java:2081) at java.util.concurrent.LinkedBlockingQueue.take(java.base@11.0.14/LinkedBlockingQueue.java:433) at java.util.concurrent.ThreadPoolExecutor.getTask(java.base@11.0.14/ThreadPoolExecutor.java:1054) at java.util.concurrent.ThreadPoolExecutor.runWorker(java.base@11.0.14/ThreadPoolExecutor.java:1114) at java.util.concurrent.ThreadPoolExecutor$Worker.run(java.base@11.0.14/ThreadPoolExecutor.java:628) at java.lang.Thread.run(java.base@11.0.14/Thread.java:829) "NIOWorkerThread-11" #207 daemon prio=5 os_prio=0 cpu=25.60ms elapsed=6574.07s tid=0x00007fd134015800 nid=0x8b9d waiting on condition [0x00007fd090bfe000] java.lang.Thread.State: WAITING (parking) at jdk.internal.misc.Unsafe.park(java.base@11.0.14/Native Method) - parking to wait for <0x000000077b7cb848> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.park(java.base@11.0.14/LockSupport.java:194) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(java.base@11.0.14/AbstractQueuedSynchronizer.java:2081) at java.util.concurrent.LinkedBlockingQueue.take(java.base@11.0.14/LinkedBlockingQueue.java:433) at java.util.concurrent.ThreadPoolExecutor.getTask(java.base@11.0.14/ThreadPoolExecutor.java:1054) at java.util.concurrent.ThreadPoolExecutor.runWorker(java.base@11.0.14/ThreadPoolExecutor.java:1114) at java.util.concurrent.ThreadPoolExecutor$Worker.run(java.base@11.0.14/ThreadPoolExecutor.java:628) at java.lang.Thread.run(java.base@11.0.14/Thread.java:829) "NIOWorkerThread-12" #208 daemon prio=5 os_prio=0 cpu=24.54ms elapsed=6574.07s tid=0x00007fd134017800 nid=0x8b9e waiting on condition [0x00007fd090afe000] java.lang.Thread.State: WAITING (parking) at jdk.internal.misc.Unsafe.park(java.base@11.0.14/Native Method) - parking to wait for <0x000000077b7cb848> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.park(java.base@11.0.14/LockSupport.java:194) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(java.base@11.0.14/AbstractQueuedSynchronizer.java:2081) at java.util.concurrent.LinkedBlockingQueue.take(java.base@11.0.14/LinkedBlockingQueue.java:433) at java.util.concurrent.ThreadPoolExecutor.getTask(java.base@11.0.14/ThreadPoolExecutor.java:1054) at java.util.concurrent.ThreadPoolExecutor.runWorker(java.base@11.0.14/ThreadPoolExecutor.java:1114) at java.util.concurrent.ThreadPoolExecutor$Worker.run(java.base@11.0.14/ThreadPoolExecutor.java:628) at java.lang.Thread.run(java.base@11.0.14/Thread.java:829) "NIOWorkerThread-13" #209 daemon prio=5 os_prio=0 cpu=23.20ms elapsed=6574.07s tid=0x00007fd134019000 nid=0x8b9f waiting on condition [0x00007fd0909fe000] java.lang.Thread.State: WAITING (parking) at jdk.internal.misc.Unsafe.park(java.base@11.0.14/Native Method) - parking to wait for <0x000000077b7cb848> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.park(java.base@11.0.14/LockSupport.java:194) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(java.base@11.0.14/AbstractQueuedSynchronizer.java:2081) at java.util.concurrent.LinkedBlockingQueue.take(java.base@11.0.14/LinkedBlockingQueue.java:433) at java.util.concurrent.ThreadPoolExecutor.getTask(java.base@11.0.14/ThreadPoolExecutor.java:1054) at java.util.concurrent.ThreadPoolExecutor.runWorker(java.base@11.0.14/ThreadPoolExecutor.java:1114) at java.util.concurrent.ThreadPoolExecutor$Worker.run(java.base@11.0.14/ThreadPoolExecutor.java:628) at java.lang.Thread.run(java.base@11.0.14/Thread.java:829) "NIOWorkerThread-14" #210 daemon prio=5 os_prio=0 cpu=23.04ms elapsed=6574.07s tid=0x00007fd13401b000 nid=0x8ba0 waiting on condition [0x00007fd0908fe000] java.lang.Thread.State: WAITING (parking) at jdk.internal.misc.Unsafe.park(java.base@11.0.14/Native Method) - parking to wait for <0x000000077b7cb848> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.park(java.base@11.0.14/LockSupport.java:194) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(java.base@11.0.14/AbstractQueuedSynchronizer.java:2081) at java.util.concurrent.LinkedBlockingQueue.take(java.base@11.0.14/LinkedBlockingQueue.java:433) at java.util.concurrent.ThreadPoolExecutor.getTask(java.base@11.0.14/ThreadPoolExecutor.java:1054) at java.util.concurrent.ThreadPoolExecutor.runWorker(java.base@11.0.14/ThreadPoolExecutor.java:1114) at java.util.concurrent.ThreadPoolExecutor$Worker.run(java.base@11.0.14/ThreadPoolExecutor.java:628) at java.lang.Thread.run(java.base@11.0.14/Thread.java:829) "NIOWorkerThread-15" #211 daemon prio=5 os_prio=0 cpu=23.91ms elapsed=6574.07s tid=0x00007fd13401c800 nid=0x8ba1 waiting on condition [0x00007fd0907fe000] java.lang.Thread.State: WAITING (parking) at jdk.internal.misc.Unsafe.park(java.base@11.0.14/Native Method) - parking to wait for <0x000000077b7cb848> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.park(java.base@11.0.14/LockSupport.java:194) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(java.base@11.0.14/AbstractQueuedSynchronizer.java:2081) at java.util.concurrent.LinkedBlockingQueue.take(java.base@11.0.14/LinkedBlockingQueue.java:433) at java.util.concurrent.ThreadPoolExecutor.getTask(java.base@11.0.14/ThreadPoolExecutor.java:1054) at java.util.concurrent.ThreadPoolExecutor.runWorker(java.base@11.0.14/ThreadPoolExecutor.java:1114) at java.util.concurrent.ThreadPoolExecutor$Worker.run(java.base@11.0.14/ThreadPoolExecutor.java:628) at java.lang.Thread.run(java.base@11.0.14/Thread.java:829) "NIOWorkerThread-16" #212 daemon prio=5 os_prio=0 cpu=25.08ms elapsed=6574.07s tid=0x00007fd13401e800 nid=0x8ba2 waiting on condition [0x00007fd0906fe000] java.lang.Thread.State: WAITING (parking) at jdk.internal.misc.Unsafe.park(java.base@11.0.14/Native Method) - parking to wait for <0x000000077b7cb848> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.park(java.base@11.0.14/LockSupport.java:194) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(java.base@11.0.14/AbstractQueuedSynchronizer.java:2081) at java.util.concurrent.LinkedBlockingQueue.take(java.base@11.0.14/LinkedBlockingQueue.java:433) at java.util.concurrent.ThreadPoolExecutor.getTask(java.base@11.0.14/ThreadPoolExecutor.java:1054) at java.util.concurrent.ThreadPoolExecutor.runWorker(java.base@11.0.14/ThreadPoolExecutor.java:1114) at java.util.concurrent.ThreadPoolExecutor$Worker.run(java.base@11.0.14/ThreadPoolExecutor.java:628) at java.lang.Thread.run(java.base@11.0.14/Thread.java:829) "RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44617" #213 daemon prio=5 os_prio=0 cpu=0.65ms elapsed=6574.04s tid=0x00007fd124846000 nid=0x8ba3 in Object.wait() [0x00007fd0905fe000] java.lang.Thread.State: WAITING (on object monitor) at java.lang.Object.wait(java.base@11.0.14/Native Method) - waiting on at java.lang.Object.wait(java.base@11.0.14/Object.java:328) at org.apache.hadoop.hbase.quotas.MasterQuotaManager$NamedLock.lock(MasterQuotaManager.java:633) - waiting to re-lock in wait() <0x000000077caebc38> (a java.util.HashSet) at org.apache.hadoop.hbase.quotas.MasterQuotaManager.setQuota(MasterQuotaManager.java:138) at org.apache.hadoop.hbase.master.MasterRpcServices.setQuota(MasterRpcServices.java:1706) at org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$2.callBlockingMethod(MasterProtos.java) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:385) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) "RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44617" #214 daemon prio=5 os_prio=0 cpu=53.33ms elapsed=6574.04s tid=0x00007fd12484e000 nid=0x8ba4 in Object.wait() [0x00007fd0904fe000] java.lang.Thread.State: WAITING (on object monitor) at java.lang.Object.wait(java.base@11.0.14/Native Method) - waiting on at java.lang.Object.wait(java.base@11.0.14/Object.java:328) at org.apache.hadoop.hbase.quotas.MasterQuotaManager$NamedLock.lock(MasterQuotaManager.java:633) - waiting to re-lock in wait() <0x000000077caebc38> (a java.util.HashSet) at org.apache.hadoop.hbase.quotas.MasterQuotaManager.setQuota(MasterQuotaManager.java:138) at org.apache.hadoop.hbase.master.MasterRpcServices.setQuota(MasterRpcServices.java:1706) at org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$2.callBlockingMethod(MasterProtos.java) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:385) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) "RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44617" #215 daemon prio=5 os_prio=0 cpu=277.63ms elapsed=6574.04s tid=0x00007fd12484f800 nid=0x8ba5 in Object.wait() [0x00007fd0903fd000] java.lang.Thread.State: TIMED_WAITING (on object monitor) at java.lang.Object.wait(java.base@11.0.14/Native Method) - waiting on at org.apache.hadoop.hbase.client.RpcRetryingCallerImpl.callWithRetries(RpcRetryingCallerImpl.java:168) - waiting to re-lock in wait() <0x0000000781b93400> (a java.util.concurrent.atomic.AtomicBoolean) at org.apache.hadoop.hbase.client.HTable.get(HTable.java:385) at org.apache.hadoop.hbase.client.HTable.lambda$get$0(HTable.java:358) at org.apache.hadoop.hbase.client.HTable$$Lambda$576/0x0000000800b30040.call(Unknown Source) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:185) at org.apache.hadoop.hbase.client.HTable.get(HTable.java:358) at org.apache.hadoop.hbase.MetaTableAccessor.getTableState(MetaTableAccessor.java:1120) at org.apache.hadoop.hbase.client.ConnectionImplementation.getTableState(ConnectionImplementation.java:2162) at org.apache.hadoop.hbase.client.ConnectionImplementation.isTableDisabled(ConnectionImplementation.java:694) at org.apache.hadoop.hbase.client.RegionServerCallable.prepare(RegionServerCallable.java:218) at org.apache.hadoop.hbase.client.RpcRetryingCallerImpl.callWithRetries(RpcRetryingCallerImpl.java:103) at org.apache.hadoop.hbase.client.HTable.get(HTable.java:385) at org.apache.hadoop.hbase.client.HTable.lambda$get$0(HTable.java:358) at org.apache.hadoop.hbase.client.HTable$$Lambda$576/0x0000000800b30040.call(Unknown Source) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:185) at org.apache.hadoop.hbase.client.HTable.get(HTable.java:358) at org.apache.hadoop.hbase.quotas.QuotaTableUtil.doGet(QuotaTableUtil.java:903) at org.apache.hadoop.hbase.quotas.QuotaTableUtil.getQuotas(QuotaTableUtil.java:202) at org.apache.hadoop.hbase.quotas.QuotaTableUtil.getQuotas(QuotaTableUtil.java:190) at org.apache.hadoop.hbase.quotas.QuotaTableUtil.getUserQuota(QuotaTableUtil.java:174) at org.apache.hadoop.hbase.quotas.MasterQuotaManager$1.fetch(MasterQuotaManager.java:185) at org.apache.hadoop.hbase.quotas.MasterQuotaManager.setQuota(MasterQuotaManager.java:482) at org.apache.hadoop.hbase.quotas.MasterQuotaManager.setUserQuota(MasterQuotaManager.java:181) at org.apache.hadoop.hbase.quotas.MasterQuotaManager.setQuota(MasterQuotaManager.java:145) at org.apache.hadoop.hbase.master.MasterRpcServices.setQuota(MasterRpcServices.java:1706) at org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$2.callBlockingMethod(MasterProtos.java) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:385) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) "RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=44617" #216 daemon prio=5 os_prio=0 cpu=72.08ms elapsed=6574.04s tid=0x00007fd124851800 nid=0x8ba6 waiting on condition [0x00007fd0902fe000] java.lang.Thread.State: WAITING (parking) at jdk.internal.misc.Unsafe.park(java.base@11.0.14/Native Method) - parking to wait for <0x000000077b7bcce0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.park(java.base@11.0.14/LockSupport.java:194) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(java.base@11.0.14/AbstractQueuedSynchronizer.java:2081) at java.util.concurrent.LinkedBlockingQueue.take(java.base@11.0.14/LinkedBlockingQueue.java:433) at org.apache.hadoop.hbase.ipc.RpcHandler.getCallRunner(RpcHandler.java:71) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) "RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44617" #217 daemon prio=5 os_prio=0 cpu=0.23ms elapsed=6574.04s tid=0x00007fd124853000 nid=0x8ba7 waiting on condition [0x00007fd0901fe000] java.lang.Thread.State: WAITING (parking) at jdk.internal.misc.Unsafe.park(java.base@11.0.14/Native Method) - parking to wait for <0x000000077b816138> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.park(java.base@11.0.14/LockSupport.java:194) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(java.base@11.0.14/AbstractQueuedSynchronizer.java:2081) at java.util.concurrent.LinkedBlockingQueue.take(java.base@11.0.14/LinkedBlockingQueue.java:433) at org.apache.hadoop.hbase.ipc.RpcHandler.getCallRunner(RpcHandler.java:71) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) "RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44617" #218 daemon prio=5 os_prio=0 cpu=0.15ms elapsed=6574.04s tid=0x00007fd124855000 nid=0x8ba8 waiting on condition [0x00007fd0900fe000] java.lang.Thread.State: WAITING (parking) at jdk.internal.misc.Unsafe.park(java.base@11.0.14/Native Method) - parking to wait for <0x000000077b816138> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.park(java.base@11.0.14/LockSupport.java:194) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(java.base@11.0.14/AbstractQueuedSynchronizer.java:2081) at java.util.concurrent.LinkedBlockingQueue.take(java.base@11.0.14/LinkedBlockingQueue.java:433) at org.apache.hadoop.hbase.ipc.RpcHandler.getCallRunner(RpcHandler.java:71) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) "RpcServer.replication.FPBQ.Fifo.handler=0,queue=0,port=44617" #219 daemon prio=5 os_prio=0 cpu=0.19ms elapsed=6574.04s tid=0x00007fd124856800 nid=0x8ba9 waiting on condition [0x00007fd08fffe000] java.lang.Thread.State: WAITING (parking) at jdk.internal.misc.Unsafe.park(java.base@11.0.14/Native Method) - parking to wait for <0x000000077b816360> (a java.util.concurrent.Semaphore$NonfairSync) at java.util.concurrent.locks.LockSupport.park(java.base@11.0.14/LockSupport.java:194) at java.util.concurrent.locks.AbstractQueuedSynchronizer.parkAndCheckInterrupt(java.base@11.0.14/AbstractQueuedSynchronizer.java:885) at java.util.concurrent.locks.AbstractQueuedSynchronizer.doAcquireSharedInterruptibly(java.base@11.0.14/AbstractQueuedSynchronizer.java:1039) at java.util.concurrent.locks.AbstractQueuedSynchronizer.acquireSharedInterruptibly(java.base@11.0.14/AbstractQueuedSynchronizer.java:1345) at java.util.concurrent.Semaphore.acquire(java.base@11.0.14/Semaphore.java:318) at org.apache.hadoop.hbase.ipc.FastPathRpcHandler.getCallRunner(FastPathRpcHandler.java:55) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) "RpcServer.replication.FPBQ.Fifo.handler=1,queue=0,port=44617" #220 daemon prio=5 os_prio=0 cpu=0.35ms elapsed=6574.04s tid=0x00007fd124858800 nid=0x8baa waiting on condition [0x00007fd08fefe000] java.lang.Thread.State: WAITING (parking) at jdk.internal.misc.Unsafe.park(java.base@11.0.14/Native Method) - parking to wait for <0x000000077b7bd098> (a java.util.concurrent.Semaphore$NonfairSync) at java.util.concurrent.locks.LockSupport.park(java.base@11.0.14/LockSupport.java:194) at java.util.concurrent.locks.AbstractQueuedSynchronizer.parkAndCheckInterrupt(java.base@11.0.14/AbstractQueuedSynchronizer.java:885) at java.util.concurrent.locks.AbstractQueuedSynchronizer.doAcquireSharedInterruptibly(java.base@11.0.14/AbstractQueuedSynchronizer.java:1039) at java.util.concurrent.locks.AbstractQueuedSynchronizer.acquireSharedInterruptibly(java.base@11.0.14/AbstractQueuedSynchronizer.java:1345) at java.util.concurrent.Semaphore.acquire(java.base@11.0.14/Semaphore.java:318) at org.apache.hadoop.hbase.ipc.FastPathRpcHandler.getCallRunner(FastPathRpcHandler.java:55) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) "RpcServer.replication.FPBQ.Fifo.handler=2,queue=0,port=44617" #221 daemon prio=5 os_prio=0 cpu=0.16ms elapsed=6574.04s tid=0x00007fd12485a000 nid=0x8bab waiting on condition [0x00007fd08fdfe000] java.lang.Thread.State: WAITING (parking) at jdk.internal.misc.Unsafe.park(java.base@11.0.14/Native Method) - parking to wait for <0x000000077b816570> (a java.util.concurrent.Semaphore$NonfairSync) at java.util.concurrent.locks.LockSupport.park(java.base@11.0.14/LockSupport.java:194) at java.util.concurrent.locks.AbstractQueuedSynchronizer.parkAndCheckInterrupt(java.base@11.0.14/AbstractQueuedSynchronizer.java:885) at java.util.concurrent.locks.AbstractQueuedSynchronizer.doAcquireSharedInterruptibly(java.base@11.0.14/AbstractQueuedSynchronizer.java:1039) at java.util.concurrent.locks.AbstractQueuedSynchronizer.acquireSharedInterruptibly(java.base@11.0.14/AbstractQueuedSynchronizer.java:1345) at java.util.concurrent.Semaphore.acquire(java.base@11.0.14/Semaphore.java:318) at org.apache.hadoop.hbase.ipc.FastPathRpcHandler.getCallRunner(FastPathRpcHandler.java:55) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) "RpcServer.metaPriority.FPBQ.Fifo.handler=0,queue=0,port=44617" #222 daemon prio=5 os_prio=0 cpu=2.94ms elapsed=6574.04s tid=0x00007fd12485c000 nid=0x8bac waiting on condition [0x00007fd08fcfe000] java.lang.Thread.State: WAITING (parking) at jdk.internal.misc.Unsafe.park(java.base@11.0.14/Native Method) - parking to wait for <0x000000077b816908> (a java.util.concurrent.Semaphore$NonfairSync) at java.util.concurrent.locks.LockSupport.park(java.base@11.0.14/LockSupport.java:194) at java.util.concurrent.locks.AbstractQueuedSynchronizer.parkAndCheckInterrupt(java.base@11.0.14/AbstractQueuedSynchronizer.java:885) at java.util.concurrent.locks.AbstractQueuedSynchronizer.doAcquireSharedInterruptibly(java.base@11.0.14/AbstractQueuedSynchronizer.java:1039) at java.util.concurrent.locks.AbstractQueuedSynchronizer.acquireSharedInterruptibly(java.base@11.0.14/AbstractQueuedSynchronizer.java:1345) at java.util.concurrent.Semaphore.acquire(java.base@11.0.14/Semaphore.java:318) at org.apache.hadoop.hbase.ipc.FastPathRpcHandler.getCallRunner(FastPathRpcHandler.java:55) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) "RS-EventLoopGroup-3-1" #226 daemon prio=10 os_prio=0 cpu=25.65ms elapsed=6573.94s tid=0x00007fd124604800 nid=0x8bae runnable [0x00007fd08fbfe000] java.lang.Thread.State: RUNNABLE at org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) at org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:193) at org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:186) at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:290) at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:347) at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:986) at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) at java.lang.Thread.run(java.base@11.0.14/Thread.java:829) "RS-EventLoopGroup-4-1" #248 daemon prio=10 os_prio=0 cpu=3172.18ms elapsed=6573.89s tid=0x00007fd124631800 nid=0x8bc0 runnable [0x00007fd08e8fe000] java.lang.Thread.State: RUNNABLE at org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) at org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:193) at org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:186) at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:290) at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:347) at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:986) at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) at java.lang.Thread.run(java.base@11.0.14/Thread.java:829) "M:0;zhangduo-ubuntu:44617" #224 daemon prio=5 os_prio=0 cpu=727.70ms elapsed=6573.83s tid=0x00007fd12470f000 nid=0x8bd2 in Object.wait() [0x00007fd08d6fe000] java.lang.Thread.State: TIMED_WAITING (on object monitor) at java.lang.Object.wait(java.base@11.0.14/Native Method) - waiting on at org.apache.hadoop.hbase.util.Sleeper.sleep(Sleeper.java:81) - waiting to re-lock in wait() <0x000000077babed90> (a java.lang.Object) at org.apache.hadoop.hbase.util.Sleeper.sleep(Sleeper.java:64) at org.apache.hadoop.hbase.master.HMaster.waitForMasterActive(HMaster.java:647) at org.apache.hadoop.hbase.regionserver.HRegionServer.initializeZooKeeper(HRegionServer.java:978) at org.apache.hadoop.hbase.regionserver.HRegionServer.preRegistrationInitialization(HRegionServer.java:922) at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:1030) at org.apache.hadoop.hbase.master.HMaster.run(HMaster.java:554) at java.lang.Thread.run(java.base@11.0.14/Thread.java:829) "Monitor thread for TaskMonitor" #271 daemon prio=5 os_prio=0 cpu=23.30ms elapsed=6573.80s tid=0x00007fd18001d000 nid=0x8bd4 waiting on condition [0x00007fd08d7fe000] java.lang.Thread.State: TIMED_WAITING (sleeping) at java.lang.Thread.sleep(java.base@11.0.14/Native Method) at org.apache.hadoop.hbase.monitoring.TaskMonitor$MonitorRunnable.run(TaskMonitor.java:296) at java.lang.Thread.run(java.base@11.0.14/Thread.java:829) "master/zhangduo-ubuntu:0:becomeActiveMaster-MemStoreChunkPool Statistics" #273 daemon prio=5 os_prio=0 cpu=6.00ms elapsed=6573.78s tid=0x00007fd180023800 nid=0x8bd5 waiting on condition [0x00007fd08d4fe000] java.lang.Thread.State: TIMED_WAITING (parking) at jdk.internal.misc.Unsafe.park(java.base@11.0.14/Native Method) - parking to wait for <0x000000077b94dad0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(java.base@11.0.14/LockSupport.java:234) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(java.base@11.0.14/AbstractQueuedSynchronizer.java:2123) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(java.base@11.0.14/ScheduledThreadPoolExecutor.java:1182) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(java.base@11.0.14/ScheduledThreadPoolExecutor.java:899) at java.util.concurrent.ThreadPoolExecutor.getTask(java.base@11.0.14/ThreadPoolExecutor.java:1054) at java.util.concurrent.ThreadPoolExecutor.runWorker(java.base@11.0.14/ThreadPoolExecutor.java:1114) at java.util.concurrent.ThreadPoolExecutor$Worker.run(java.base@11.0.14/ThreadPoolExecutor.java:628) at java.lang.Thread.run(java.base@11.0.14/Thread.java:829) "master/zhangduo-ubuntu:0:becomeActiveMaster-MemStoreChunkPool Statistics" #275 daemon prio=5 os_prio=0 cpu=1.86ms elapsed=6573.78s tid=0x00007fd180025000 nid=0x8bd6 waiting on condition [0x00007fd08d3fe000] java.lang.Thread.State: TIMED_WAITING (parking) at jdk.internal.misc.Unsafe.park(java.base@11.0.14/Native Method) - parking to wait for <0x000000077b94e3b8> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(java.base@11.0.14/LockSupport.java:234) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(java.base@11.0.14/AbstractQueuedSynchronizer.java:2123) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(java.base@11.0.14/ScheduledThreadPoolExecutor.java:1182) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(java.base@11.0.14/ScheduledThreadPoolExecutor.java:899) at java.util.concurrent.ThreadPoolExecutor.getTask(java.base@11.0.14/ThreadPoolExecutor.java:1054) at java.util.concurrent.ThreadPoolExecutor.runWorker(java.base@11.0.14/ThreadPoolExecutor.java:1114) at java.util.concurrent.ThreadPoolExecutor$Worker.run(java.base@11.0.14/ThreadPoolExecutor.java:628) at java.lang.Thread.run(java.base@11.0.14/Thread.java:829) "org.apache.hadoop.hdfs.PeerCache@580ec87a" #277 daemon prio=5 os_prio=0 cpu=85.56ms elapsed=6573.72s tid=0x00007fd18002f000 nid=0x8bd8 waiting on condition [0x00007fd08d1fe000] java.lang.Thread.State: TIMED_WAITING (sleeping) at java.lang.Thread.sleep(java.base@11.0.14/Native Method) at org.apache.hadoop.hdfs.PeerCache.run(PeerCache.java:253) at org.apache.hadoop.hdfs.PeerCache.access$000(PeerCache.java:46) at org.apache.hadoop.hdfs.PeerCache$1.run(PeerCache.java:124) at java.lang.Thread.run(java.base@11.0.14/Thread.java:829) "master:store-WAL-Roller" #292 daemon prio=5 os_prio=0 cpu=791929.39ms elapsed=6573.56s tid=0x00007fd18007d800 nid=0x8be7 in Object.wait() [0x00007fd08ccfe000] java.lang.Thread.State: TIMED_WAITING (on object monitor) at java.lang.Object.wait(java.base@11.0.14/Native Method) - waiting on at org.apache.hadoop.hbase.wal.AbstractWALRoller.run(AbstractWALRoller.java:179) - waiting to re-lock in wait() <0x000000077bacc378> (a org.apache.hadoop.hbase.master.region.MasterRegionWALRoller) "RS-EventLoopGroup-4-2" #305 daemon prio=10 os_prio=0 cpu=3232.55ms elapsed=6573.36s tid=0x00007fd180134800 nid=0x8bf7 runnable [0x00007fd08cffe000] java.lang.Thread.State: RUNNABLE at org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait0(Native Method) at org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:177) at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWait(EpollEventLoop.java:281) at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:351) at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:986) at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) at java.lang.Thread.run(java.base@11.0.14/Thread.java:829) "RS-EventLoopGroup-4-3" #306 daemon prio=10 os_prio=0 cpu=3117.44ms elapsed=6573.36s tid=0x00007fd180137000 nid=0x8bf8 runnable [0x00007fd08c7fe000] java.lang.Thread.State: RUNNABLE at org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) at org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:193) at org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:186) at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:290) at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:347) at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:986) at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) at java.lang.Thread.run(java.base@11.0.14/Thread.java:829) "AsyncFSWAL-0-hdfs://zhangduo-ubuntu:40133/user/zhangduo/test-data/69c6498a-2c37-c570-8a20-bf781a62cf6e/MasterData" #311 daemon prio=5 os_prio=0 cpu=513.80ms elapsed=6573.29s tid=0x00007fd180146800 nid=0x8bfd waiting on condition [0x00007fd0894fb000] java.lang.Thread.State: WAITING (parking) at jdk.internal.misc.Unsafe.park(java.base@11.0.14/Native Method) - parking to wait for <0x000000077baafac8> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.park(java.base@11.0.14/LockSupport.java:194) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(java.base@11.0.14/AbstractQueuedSynchronizer.java:2081) at java.util.concurrent.LinkedBlockingQueue.take(java.base@11.0.14/LinkedBlockingQueue.java:433) at java.util.concurrent.ThreadPoolExecutor.getTask(java.base@11.0.14/ThreadPoolExecutor.java:1054) at java.util.concurrent.ThreadPoolExecutor.runWorker(java.base@11.0.14/ThreadPoolExecutor.java:1114) at java.util.concurrent.ThreadPoolExecutor$Worker.run(java.base@11.0.14/ThreadPoolExecutor.java:628) at java.lang.Thread.run(java.base@11.0.14/Thread.java:829) "master:store-Flusher" #314 daemon prio=5 os_prio=0 cpu=28.79ms elapsed=6573.17s tid=0x00007fd180106800 nid=0x8c01 waiting on condition [0x00007fd0893fb000] java.lang.Thread.State: TIMED_WAITING (parking) at jdk.internal.misc.Unsafe.park(java.base@11.0.14/Native Method) - parking to wait for <0x000000077baafba8> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(java.base@11.0.14/LockSupport.java:234) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(java.base@11.0.14/AbstractQueuedSynchronizer.java:2211) at org.apache.hadoop.hbase.master.region.MasterRegionFlusherAndCompactor.flushLoop(MasterRegionFlusherAndCompactor.java:193) at org.apache.hadoop.hbase.master.region.MasterRegionFlusherAndCompactor$$Lambda$316/0x0000000800979040.run(Unknown Source) at java.lang.Thread.run(java.base@11.0.14/Thread.java:829) "Idle-Rpc-Conn-Sweeper-pool-0" #316 daemon prio=5 os_prio=0 cpu=10.16ms elapsed=6573.14s tid=0x00007fd180199000 nid=0x8c02 waiting on condition [0x00007fd0890fb000] java.lang.Thread.State: TIMED_WAITING (parking) at jdk.internal.misc.Unsafe.park(java.base@11.0.14/Native Method) - parking to wait for <0x000000077baecb30> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(java.base@11.0.14/LockSupport.java:234) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(java.base@11.0.14/AbstractQueuedSynchronizer.java:2123) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(java.base@11.0.14/ScheduledThreadPoolExecutor.java:1182) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(java.base@11.0.14/ScheduledThreadPoolExecutor.java:899) at java.util.concurrent.ThreadPoolExecutor.getTask(java.base@11.0.14/ThreadPoolExecutor.java:1054) at java.util.concurrent.ThreadPoolExecutor.runWorker(java.base@11.0.14/ThreadPoolExecutor.java:1114) at java.util.concurrent.ThreadPoolExecutor$Worker.run(java.base@11.0.14/ThreadPoolExecutor.java:628) at java.lang.Thread.run(java.base@11.0.14/Thread.java:829) "ProcedureDispatcherTimeoutThread" #324 daemon prio=5 os_prio=0 cpu=21.47ms elapsed=6573.07s tid=0x00007fd1801e6000 nid=0x8c03 waiting on condition [0x00007fd088ffb000] java.lang.Thread.State: TIMED_WAITING (parking) at jdk.internal.misc.Unsafe.park(java.base@11.0.14/Native Method) - parking to wait for <0x000000077bab0060> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(java.base@11.0.14/LockSupport.java:234) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(java.base@11.0.14/AbstractQueuedSynchronizer.java:2123) at java.util.concurrent.DelayQueue.poll(java.base@11.0.14/DelayQueue.java:265) at org.apache.hadoop.hbase.procedure2.util.DelayedUtil.takeWithoutInterrupt(DelayedUtil.java:83) at org.apache.hadoop.hbase.procedure2.RemoteProcedureDispatcher$TimeoutExecutorThread.run(RemoteProcedureDispatcher.java:314) "zhangduo-ubuntu:44617" #325 daemon prio=5 os_prio=0 cpu=21.14ms elapsed=6573.06s tid=0x00007fd18020f800 nid=0x8c04 waiting on condition [0x00007fd088efb000] java.lang.Thread.State: WAITING (parking) at jdk.internal.misc.Unsafe.park(java.base@11.0.14/Native Method) - parking to wait for <0x000000077bad2af8> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.park(java.base@11.0.14/LockSupport.java:194) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(java.base@11.0.14/AbstractQueuedSynchronizer.java:2081) at org.apache.hadoop.hbase.master.assignment.AssignmentManager.waitOnAssignQueue(AssignmentManager.java:2171) at org.apache.hadoop.hbase.master.assignment.AssignmentManager.processAssignQueue(AssignmentManager.java:2193) at org.apache.hadoop.hbase.master.assignment.AssignmentManager.access$600(AssignmentManager.java:108) at org.apache.hadoop.hbase.master.assignment.AssignmentManager$1.run(AssignmentManager.java:2133) "normalizer-worker-0" #326 daemon prio=5 os_prio=0 cpu=0.24ms elapsed=6573.03s tid=0x00007fd180219800 nid=0x8c05 waiting on condition [0x00007fd088dfb000] java.lang.Thread.State: WAITING (parking) at jdk.internal.misc.Unsafe.park(java.base@11.0.14/Native Method) - parking to wait for <0x000000077bad3060> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.park(java.base@11.0.14/LockSupport.java:194) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(java.base@11.0.14/AbstractQueuedSynchronizer.java:2081) at org.apache.hadoop.hbase.master.normalizer.RegionNormalizerWorkQueue.take(RegionNormalizerWorkQueue.java:189) at org.apache.hadoop.hbase.master.normalizer.RegionNormalizerWorker.run(RegionNormalizerWorker.java:172) at java.util.concurrent.Executors$RunnableAdapter.call(java.base@11.0.14/Executors.java:515) at java.util.concurrent.FutureTask.run(java.base@11.0.14/FutureTask.java:264) at java.util.concurrent.ThreadPoolExecutor.runWorker(java.base@11.0.14/ThreadPoolExecutor.java:1128) at java.util.concurrent.ThreadPoolExecutor$Worker.run(java.base@11.0.14/ThreadPoolExecutor.java:628) at java.lang.Thread.run(java.base@11.0.14/Thread.java:829) "SnapshotHandlerChoreCleaner" #327 daemon prio=5 os_prio=0 cpu=45.03ms elapsed=6572.96s tid=0x00007fd180255800 nid=0x8c08 waiting on condition [0x00007fd088afb000] java.lang.Thread.State: TIMED_WAITING (parking) at jdk.internal.misc.Unsafe.park(java.base@11.0.14/Native Method) - parking to wait for <0x000000077bada7f8> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(java.base@11.0.14/LockSupport.java:234) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(java.base@11.0.14/AbstractQueuedSynchronizer.java:2123) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(java.base@11.0.14/ScheduledThreadPoolExecutor.java:1182) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(java.base@11.0.14/ScheduledThreadPoolExecutor.java:899) at java.util.concurrent.ThreadPoolExecutor.getTask(java.base@11.0.14/ThreadPoolExecutor.java:1054) at java.util.concurrent.ThreadPoolExecutor.runWorker(java.base@11.0.14/ThreadPoolExecutor.java:1114) at java.util.concurrent.ThreadPoolExecutor$Worker.run(java.base@11.0.14/ThreadPoolExecutor.java:628) at java.lang.Thread.run(java.base@11.0.14/Thread.java:829) "RpcClient-timer-pool-0" #315 daemon prio=5 os_prio=0 cpu=16910.68ms elapsed=6572.85s tid=0x00007fd0fc01c000 nid=0x8c17 waiting on condition [0x00007fd0872fa000] java.lang.Thread.State: TIMED_WAITING (sleeping) at java.lang.Thread.sleep(java.base@11.0.14/Native Method) at org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$Worker.waitForNextTick(HashedWheelTimer.java:600) at org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$Worker.run(HashedWheelTimer.java:496) at java.lang.Thread.run(java.base@11.0.14/Thread.java:829) "RS-EventLoopGroup-1-2" #334 daemon prio=10 os_prio=0 cpu=57.69ms elapsed=6572.85s tid=0x00007fd17c006800 nid=0x8c18 runnable [0x00007fd0871fa000] java.lang.Thread.State: RUNNABLE at org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) at org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:193) at org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:186) at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:290) at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:347) at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:986) at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) at java.lang.Thread.run(java.base@11.0.14/Thread.java:829) "RS-EventLoopGroup-1-3" #335 daemon prio=10 os_prio=0 cpu=68.83ms elapsed=6572.85s tid=0x00007fd17c009000 nid=0x8c19 runnable [0x00007fd0870fa000] java.lang.Thread.State: RUNNABLE at org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) at org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:193) at org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:186) at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:290) at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:347) at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:986) at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) at java.lang.Thread.run(java.base@11.0.14/Thread.java:829) "ProcExecTimeout" #317 daemon prio=5 os_prio=0 cpu=30.11ms elapsed=6572.82s tid=0x00007fd1802e4000 nid=0x8c1b waiting on condition [0x00007fd083ff7000] java.lang.Thread.State: TIMED_WAITING (parking) at jdk.internal.misc.Unsafe.park(java.base@11.0.14/Native Method) - parking to wait for <0x000000077bb26820> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(java.base@11.0.14/LockSupport.java:234) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(java.base@11.0.14/AbstractQueuedSynchronizer.java:2123) at java.util.concurrent.DelayQueue.poll(java.base@11.0.14/DelayQueue.java:274) at org.apache.hadoop.hbase.procedure2.util.DelayedUtil.takeWithoutInterrupt(DelayedUtil.java:83) at org.apache.hadoop.hbase.procedure2.TimeoutExecutorThread.run(TimeoutExecutorThread.java:56) "WorkerMonitor" #318 daemon prio=5 os_prio=0 cpu=30.65ms elapsed=6572.82s tid=0x00007fd1802e5800 nid=0x8c1c waiting on condition [0x00007fd083ef7000] java.lang.Thread.State: TIMED_WAITING (parking) at jdk.internal.misc.Unsafe.park(java.base@11.0.14/Native Method) - parking to wait for <0x000000077badcaf0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(java.base@11.0.14/LockSupport.java:234) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(java.base@11.0.14/AbstractQueuedSynchronizer.java:2123) at java.util.concurrent.DelayQueue.poll(java.base@11.0.14/DelayQueue.java:274) at org.apache.hadoop.hbase.procedure2.util.DelayedUtil.takeWithoutInterrupt(DelayedUtil.java:83) at org.apache.hadoop.hbase.procedure2.TimeoutExecutorThread.run(TimeoutExecutorThread.java:56) "PEWorker-1" #319 daemon prio=5 os_prio=0 cpu=92.31ms elapsed=6572.82s tid=0x00007fd1802e7800 nid=0x8c1d waiting on condition [0x00007fd083df7000] java.lang.Thread.State: TIMED_WAITING (parking) at jdk.internal.misc.Unsafe.park(java.base@11.0.14/Native Method) - parking to wait for <0x000000077bab05f0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(java.base@11.0.14/LockSupport.java:234) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(java.base@11.0.14/AbstractQueuedSynchronizer.java:2123) at org.apache.hadoop.hbase.procedure2.AbstractProcedureScheduler.poll(AbstractProcedureScheduler.java:165) at org.apache.hadoop.hbase.procedure2.AbstractProcedureScheduler.poll(AbstractProcedureScheduler.java:147) at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:1956) "PEWorker-2" #320 daemon prio=5 os_prio=0 cpu=49.56ms elapsed=6572.82s tid=0x00007fd1802e9800 nid=0x8c1e waiting on condition [0x00007fd083cf7000] java.lang.Thread.State: TIMED_WAITING (parking) at jdk.internal.misc.Unsafe.park(java.base@11.0.14/Native Method) - parking to wait for <0x000000077bab05f0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(java.base@11.0.14/LockSupport.java:234) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(java.base@11.0.14/AbstractQueuedSynchronizer.java:2123) at org.apache.hadoop.hbase.procedure2.AbstractProcedureScheduler.poll(AbstractProcedureScheduler.java:165) at org.apache.hadoop.hbase.procedure2.AbstractProcedureScheduler.poll(AbstractProcedureScheduler.java:147) at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:1956) "PEWorker-3" #321 daemon prio=5 os_prio=0 cpu=70.99ms elapsed=6572.82s tid=0x00007fd1802eb000 nid=0x8c1f waiting on condition [0x00007fd083bf7000] java.lang.Thread.State: TIMED_WAITING (parking) at jdk.internal.misc.Unsafe.park(java.base@11.0.14/Native Method) - parking to wait for <0x000000077bab05f0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(java.base@11.0.14/LockSupport.java:234) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(java.base@11.0.14/AbstractQueuedSynchronizer.java:2123) at org.apache.hadoop.hbase.procedure2.AbstractProcedureScheduler.poll(AbstractProcedureScheduler.java:165) at org.apache.hadoop.hbase.procedure2.AbstractProcedureScheduler.poll(AbstractProcedureScheduler.java:147) at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:1956) "PEWorker-4" #322 daemon prio=5 os_prio=0 cpu=77.34ms elapsed=6572.82s tid=0x00007fd1802ed000 nid=0x8c20 waiting on condition [0x00007fd083af7000] java.lang.Thread.State: TIMED_WAITING (parking) at jdk.internal.misc.Unsafe.park(java.base@11.0.14/Native Method) - parking to wait for <0x000000077bab05f0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(java.base@11.0.14/LockSupport.java:234) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(java.base@11.0.14/AbstractQueuedSynchronizer.java:2123) at org.apache.hadoop.hbase.procedure2.AbstractProcedureScheduler.poll(AbstractProcedureScheduler.java:165) at org.apache.hadoop.hbase.procedure2.AbstractProcedureScheduler.poll(AbstractProcedureScheduler.java:147) at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:1956) "PEWorker-5" #323 daemon prio=5 os_prio=0 cpu=43.78ms elapsed=6572.82s tid=0x00007fd1802ee800 nid=0x8c21 waiting on condition [0x00007fd0839f7000] java.lang.Thread.State: TIMED_WAITING (parking) at jdk.internal.misc.Unsafe.park(java.base@11.0.14/Native Method) - parking to wait for <0x000000077bab05f0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(java.base@11.0.14/LockSupport.java:234) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(java.base@11.0.14/AbstractQueuedSynchronizer.java:2123) at org.apache.hadoop.hbase.procedure2.AbstractProcedureScheduler.poll(AbstractProcedureScheduler.java:165) at org.apache.hadoop.hbase.procedure2.AbstractProcedureScheduler.poll(AbstractProcedureScheduler.java:147) at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:1956) "OldWALsCleaner-0" #338 daemon prio=5 os_prio=0 cpu=2.54ms elapsed=6572.80s tid=0x00007fd1802f9800 nid=0x8c24 waiting on condition [0x00007fd0836f7000] java.lang.Thread.State: WAITING (parking) at jdk.internal.misc.Unsafe.park(java.base@11.0.14/Native Method) - parking to wait for <0x000000077badece0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.park(java.base@11.0.14/LockSupport.java:194) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(java.base@11.0.14/AbstractQueuedSynchronizer.java:2081) at java.util.concurrent.LinkedBlockingQueue.take(java.base@11.0.14/LinkedBlockingQueue.java:433) at org.apache.hadoop.hbase.master.cleaner.LogCleaner.deleteFile(LogCleaner.java:172) at org.apache.hadoop.hbase.master.cleaner.LogCleaner.lambda$createOldWalsCleaner$1(LogCleaner.java:152) at org.apache.hadoop.hbase.master.cleaner.LogCleaner$$Lambda$446/0x0000000800a5f040.run(Unknown Source) at java.lang.Thread.run(java.base@11.0.14/Thread.java:829) "master/zhangduo-ubuntu:0.Chore.1" #340 daemon prio=5 os_prio=0 cpu=34.15ms elapsed=6572.80s tid=0x00007fd1802fd800 nid=0x8c26 in Object.wait() [0x00007fd0834f6000] java.lang.Thread.State: TIMED_WAITING (on object monitor) at java.lang.Object.wait(java.base@11.0.14/Native Method) - waiting on at java.lang.Object.wait(java.base@11.0.14/Object.java:462) at java.util.concurrent.TimeUnit.timedWait(java.base@11.0.14/TimeUnit.java:408) at org.apache.hadoop.hbase.client.ResultBoundedCompletionService.pollForSpecificCompletedTask(ResultBoundedCompletionService.java:246) - waiting to re-lock in wait() <0x00000007824b1600> (a [Lorg.apache.hadoop.hbase.client.ResultBoundedCompletionService$QueueingFuture;) at org.apache.hadoop.hbase.client.ResultBoundedCompletionService.pollForFirstSuccessfullyCompletedTask(ResultBoundedCompletionService.java:204) at org.apache.hadoop.hbase.client.ScannerCallableWithReplicas.call(ScannerCallableWithReplicas.java:235) at org.apache.hadoop.hbase.client.ScannerCallableWithReplicas.call(ScannerCallableWithReplicas.java:53) at org.apache.hadoop.hbase.client.RpcRetryingCallerImpl.callWithoutRetries(RpcRetryingCallerImpl.java:192) at org.apache.hadoop.hbase.client.ClientScanner.call(ClientScanner.java:270) at org.apache.hadoop.hbase.client.ClientScanner.loadCache(ClientScanner.java:439) at org.apache.hadoop.hbase.client.ClientScanner.nextWithSyncCache(ClientScanner.java:313) at org.apache.hadoop.hbase.client.ClientScanner.next(ClientScanner.java:611) at org.apache.hadoop.hbase.client.ConnectionImplementation.locateRegionInMeta(ConnectionImplementation.java:971) at org.apache.hadoop.hbase.client.ConnectionImplementation.locateRegion(ConnectionImplementation.java:849) at org.apache.hadoop.hbase.client.RpcRetryingCallerWithReadReplicas.getRegionLocations(RpcRetryingCallerWithReadReplicas.java:319) at org.apache.hadoop.hbase.client.ScannerCallableWithReplicas.call(ScannerCallableWithReplicas.java:157) at org.apache.hadoop.hbase.client.ScannerCallableWithReplicas.call(ScannerCallableWithReplicas.java:53) at org.apache.hadoop.hbase.client.RpcRetryingCallerImpl.callWithoutRetries(RpcRetryingCallerImpl.java:192) at org.apache.hadoop.hbase.client.ClientScanner.call(ClientScanner.java:270) at org.apache.hadoop.hbase.client.ClientScanner.loadCache(ClientScanner.java:439) at org.apache.hadoop.hbase.client.ClientScanner.nextWithSyncCache(ClientScanner.java:313) at org.apache.hadoop.hbase.client.ClientScanner.next(ClientScanner.java:611) at org.apache.hadoop.hbase.quotas.QuotaRetriever.next(QuotaRetriever.java:103) at org.apache.hadoop.hbase.quotas.QuotaRetriever$Iter.(QuotaRetriever.java:129) at org.apache.hadoop.hbase.quotas.QuotaRetriever.iterator(QuotaRetriever.java:121) at org.apache.hadoop.hbase.quotas.QuotaObserverChore.fetchAllTablesWithQuotasDefined(QuotaObserverChore.java:480) at org.apache.hadoop.hbase.quotas.QuotaObserverChore._chore(QuotaObserverChore.java:144) at org.apache.hadoop.hbase.quotas.QuotaObserverChore.chore(QuotaObserverChore.java:132) at org.apache.hadoop.hbase.ScheduledChore.run(ScheduledChore.java:159) at java.util.concurrent.Executors$RunnableAdapter.call(java.base@11.0.14/Executors.java:515) at java.util.concurrent.FutureTask.runAndReset(java.base@11.0.14/FutureTask.java:305) at java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.run(java.base@11.0.14/ScheduledThreadPoolExecutor.java:305) at org.apache.hadoop.hbase.JitterScheduledThreadPoolExecutorImpl$JitteredRunnableScheduledFuture.run(JitterScheduledThreadPoolExecutorImpl.java:107) at java.util.concurrent.ThreadPoolExecutor.runWorker(java.base@11.0.14/ThreadPoolExecutor.java:1128) at java.util.concurrent.ThreadPoolExecutor$Worker.run(java.base@11.0.14/ThreadPoolExecutor.java:628) at java.lang.Thread.run(java.base@11.0.14/Thread.java:829) "snapshot-hfile-cleaner-cache-refresher" #344 daemon prio=5 os_prio=0 cpu=6.32ms elapsed=6572.78s tid=0x00007fd180306800 nid=0x8c2a in Object.wait() [0x00007fd0831f7000] java.lang.Thread.State: TIMED_WAITING (on object monitor) at java.lang.Object.wait(java.base@11.0.14/Native Method) - waiting on at java.util.TimerThread.mainLoop(java.base@11.0.14/Timer.java:553) - waiting to re-lock in wait() <0x000000077bae15e0> (a java.util.TaskQueue) at java.util.TimerThread.run(java.base@11.0.14/Timer.java:506) "master/zhangduo-ubuntu:0:becomeActiveMaster-HFileCleaner.large.0-1652709601583" #345 daemon prio=5 os_prio=0 cpu=0.15ms elapsed=6572.78s tid=0x00007fd18030a000 nid=0x8c2b waiting on condition [0x00007fd0838f7000] java.lang.Thread.State: WAITING (parking) at jdk.internal.misc.Unsafe.park(java.base@11.0.14/Native Method) - parking to wait for <0x000000077bb8b138> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.park(java.base@11.0.14/LockSupport.java:194) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(java.base@11.0.14/AbstractQueuedSynchronizer.java:2081) at org.apache.hadoop.hbase.util.StealJobQueue.take(StealJobQueue.java:101) at org.apache.hadoop.hbase.master.cleaner.HFileCleaner.consumerLoop(HFileCleaner.java:268) at org.apache.hadoop.hbase.master.cleaner.HFileCleaner$1.run(HFileCleaner.java:237) "master/zhangduo-ubuntu:0:becomeActiveMaster-HFileCleaner.small.0-1652709601584" #346 daemon prio=5 os_prio=0 cpu=0.14ms elapsed=6572.78s tid=0x00007fd18030c800 nid=0x8c2c waiting on condition [0x00007fd0837f7000] java.lang.Thread.State: WAITING (parking) at jdk.internal.misc.Unsafe.park(java.base@11.0.14/Native Method) - parking to wait for <0x000000077bae1bc8> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.park(java.base@11.0.14/LockSupport.java:194) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(java.base@11.0.14/AbstractQueuedSynchronizer.java:2081) at java.util.concurrent.PriorityBlockingQueue.take(java.base@11.0.14/PriorityBlockingQueue.java:546) at org.apache.hadoop.hbase.master.cleaner.HFileCleaner.consumerLoop(HFileCleaner.java:268) at org.apache.hadoop.hbase.master.cleaner.HFileCleaner$2.run(HFileCleaner.java:252) "RegionServerTracker-0" #356 daemon prio=5 os_prio=0 cpu=12.34ms elapsed=6572.65s tid=0x00007fd18800b800 nid=0x8c3a waiting on condition [0x00007fd0832f7000] java.lang.Thread.State: WAITING (parking) at jdk.internal.misc.Unsafe.park(java.base@11.0.14/Native Method) - parking to wait for <0x000000077bb728c8> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.park(java.base@11.0.14/LockSupport.java:194) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(java.base@11.0.14/AbstractQueuedSynchronizer.java:2081) at java.util.concurrent.LinkedBlockingQueue.take(java.base@11.0.14/LinkedBlockingQueue.java:433) at java.util.concurrent.ThreadPoolExecutor.getTask(java.base@11.0.14/ThreadPoolExecutor.java:1054) at java.util.concurrent.ThreadPoolExecutor.runWorker(java.base@11.0.14/ThreadPoolExecutor.java:1114) at java.util.concurrent.ThreadPoolExecutor$Worker.run(java.base@11.0.14/ThreadPoolExecutor.java:628) at java.lang.Thread.run(java.base@11.0.14/Thread.java:829) "regionserver/zhangduo-ubuntu:0.procedureResultReporter" #370 daemon prio=5 os_prio=0 cpu=2.81ms elapsed=6572.59s tid=0x00007fd14005c800 nid=0x8c47 waiting on condition [0x00007fd0824f7000] java.lang.Thread.State: WAITING (parking) at jdk.internal.misc.Unsafe.park(java.base@11.0.14/Native Method) - parking to wait for <0x000000077bb9b970> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.park(java.base@11.0.14/LockSupport.java:194) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(java.base@11.0.14/AbstractQueuedSynchronizer.java:2081) at java.util.concurrent.LinkedBlockingQueue.take(java.base@11.0.14/LinkedBlockingQueue.java:433) at org.apache.hadoop.hbase.regionserver.RemoteProcedureResultReporter.run(RemoteProcedureResultReporter.java:74) "regionserver/zhangduo-ubuntu:0.procedureResultReporter" #371 daemon prio=5 os_prio=0 cpu=0.26ms elapsed=6572.59s tid=0x00007fd14c05f800 nid=0x8c49 waiting on condition [0x00007fd0822f7000] java.lang.Thread.State: WAITING (parking) at jdk.internal.misc.Unsafe.park(java.base@11.0.14/Native Method) - parking to wait for <0x000000077bb9bc10> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.park(java.base@11.0.14/LockSupport.java:194) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(java.base@11.0.14/AbstractQueuedSynchronizer.java:2081) at java.util.concurrent.LinkedBlockingQueue.take(java.base@11.0.14/LinkedBlockingQueue.java:433) at org.apache.hadoop.hbase.regionserver.RemoteProcedureResultReporter.run(RemoteProcedureResultReporter.java:74) "RS-EventLoopGroup-3-2" #393 daemon prio=10 os_prio=0 cpu=21.03ms elapsed=6572.06s tid=0x00007fd0e400b000 nid=0x8c5d runnable [0x00007fd07f0f5000] java.lang.Thread.State: RUNNABLE at org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) at org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:193) at org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:186) at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:290) at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:347) at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:986) at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) at java.lang.Thread.run(java.base@11.0.14/Thread.java:829) "RS-EventLoopGroup-3-3" #402 daemon prio=10 os_prio=0 cpu=30.77ms elapsed=6571.85s tid=0x00007fd0e400d000 nid=0x8c66 runnable [0x00007fd07b6f2000] java.lang.Thread.State: RUNNABLE at org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) at org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:193) at org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:186) at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:290) at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:347) at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:986) at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) at java.lang.Thread.run(java.base@11.0.14/Thread.java:829) "region-location-0" #437 daemon prio=5 os_prio=0 cpu=2.25ms elapsed=6571.13s tid=0x00007fd180361000 nid=0x8c8c waiting on condition [0x00007fd078af0000] java.lang.Thread.State: WAITING (parking) at jdk.internal.misc.Unsafe.park(java.base@11.0.14/Native Method) - parking to wait for <0x000000077badf6e0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.park(java.base@11.0.14/LockSupport.java:194) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(java.base@11.0.14/AbstractQueuedSynchronizer.java:2081) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(java.base@11.0.14/ScheduledThreadPoolExecutor.java:1170) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(java.base@11.0.14/ScheduledThreadPoolExecutor.java:899) at java.util.concurrent.ThreadPoolExecutor.getTask(java.base@11.0.14/ThreadPoolExecutor.java:1054) at java.util.concurrent.ThreadPoolExecutor.runWorker(java.base@11.0.14/ThreadPoolExecutor.java:1114) at java.util.concurrent.ThreadPoolExecutor$Worker.run(java.base@11.0.14/ThreadPoolExecutor.java:628) at java.lang.Thread.run(java.base@11.0.14/Thread.java:829) "region-location-1" #439 daemon prio=5 os_prio=0 cpu=2.96ms elapsed=6571.13s tid=0x00007fd180361800 nid=0x8c8e waiting on condition [0x00007fd0784f0000] java.lang.Thread.State: WAITING (parking) at jdk.internal.misc.Unsafe.park(java.base@11.0.14/Native Method) - parking to wait for <0x000000077badf6e0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.park(java.base@11.0.14/LockSupport.java:194) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(java.base@11.0.14/AbstractQueuedSynchronizer.java:2081) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(java.base@11.0.14/ScheduledThreadPoolExecutor.java:1170) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(java.base@11.0.14/ScheduledThreadPoolExecutor.java:899) at java.util.concurrent.ThreadPoolExecutor.getTask(java.base@11.0.14/ThreadPoolExecutor.java:1054) at java.util.concurrent.ThreadPoolExecutor.runWorker(java.base@11.0.14/ThreadPoolExecutor.java:1114) at java.util.concurrent.ThreadPoolExecutor$Worker.run(java.base@11.0.14/ThreadPoolExecutor.java:628) at java.lang.Thread.run(java.base@11.0.14/Thread.java:829) "ReadOnlyZKClient-127.0.0.1:59087@0x6eed0e50" #441 daemon prio=5 os_prio=0 cpu=34.38ms elapsed=6571.10s tid=0x00007fd124f12000 nid=0x8c90 waiting on condition [0x00007fd079df1000] java.lang.Thread.State: TIMED_WAITING (parking) at jdk.internal.misc.Unsafe.park(java.base@11.0.14/Native Method) - parking to wait for <0x000000077c4e0448> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(java.base@11.0.14/LockSupport.java:234) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(java.base@11.0.14/AbstractQueuedSynchronizer.java:2123) at java.util.concurrent.DelayQueue.poll(java.base@11.0.14/DelayQueue.java:265) at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:324) at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$$Lambda$587/0x0000000800b35c40.run(Unknown Source) at java.lang.Thread.run(java.base@11.0.14/Thread.java:829) "RPCClient-NioEventLoopGroup-5-1" #445 daemon prio=5 os_prio=0 cpu=32.63ms elapsed=6571.08s tid=0x00007fd120015000 nid=0x8c94 runnable [0x00007fd0889fb000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPoll.wait(java.base@11.0.14/Native Method) at sun.nio.ch.EPollSelectorImpl.doSelect(java.base@11.0.14/EPollSelectorImpl.java:120) at sun.nio.ch.SelectorImpl.lockAndDoSelect(java.base@11.0.14/SelectorImpl.java:124) - locked <0x000000077c4df498> (a org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySet) - locked <0x000000077c4df440> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(java.base@11.0.14/SelectorImpl.java:141) at org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:813) at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:460) at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:986) at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) at java.lang.Thread.run(java.base@11.0.14/Thread.java:829) "RPCClient-NioEventLoopGroup-5-2" #450 daemon prio=5 os_prio=0 cpu=49.39ms elapsed=6570.73s tid=0x00007fd1247ad000 nid=0x8c99 runnable [0x00007fd079ff1000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPoll.wait(java.base@11.0.14/Native Method) at sun.nio.ch.EPollSelectorImpl.doSelect(java.base@11.0.14/EPollSelectorImpl.java:120) at sun.nio.ch.SelectorImpl.lockAndDoSelect(java.base@11.0.14/SelectorImpl.java:124) - locked <0x000000077c4bde68> (a org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySet) - locked <0x000000077c4bde10> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(java.base@11.0.14/SelectorImpl.java:141) at org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:813) at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:460) at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:986) at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) at java.lang.Thread.run(java.base@11.0.14/Thread.java:829) "RPCClient-NioEventLoopGroup-5-3" #471 daemon prio=5 os_prio=0 cpu=12.62ms elapsed=6569.90s tid=0x00007fd13c006000 nid=0x8caf runnable [0x00007fd075cee000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPoll.wait(java.base@11.0.14/Native Method) at sun.nio.ch.EPollSelectorImpl.doSelect(java.base@11.0.14/EPollSelectorImpl.java:120) at sun.nio.ch.SelectorImpl.lockAndDoSelect(java.base@11.0.14/SelectorImpl.java:124) - locked <0x000000077c4db870> (a org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySet) - locked <0x000000077c4db818> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(java.base@11.0.14/SelectorImpl.java:141) at org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:813) at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:460) at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:986) at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) at java.lang.Thread.run(java.base@11.0.14/Thread.java:829) "Log-Archiver-0" #569 daemon prio=5 os_prio=0 cpu=2533839.35ms elapsed=6567.05s tid=0x00007fd0d0023000 nid=0x8d19 in Object.wait() [0x00007fd08cdfd000] java.lang.Thread.State: WAITING (on object monitor) at java.lang.Object.wait(java.base@11.0.14/Native Method) - waiting on at java.lang.Object.wait(java.base@11.0.14/Object.java:328) at org.apache.hadoop.util.concurrent.AsyncGet$Util.wait(AsyncGet.java:59) at org.apache.hadoop.ipc.Client.getRpcResponse(Client.java:1499) - waiting to re-lock in wait() <0x0000000786e10b90> (a org.apache.hadoop.ipc.Client$Call) at org.apache.hadoop.ipc.Client.call(Client.java:1457) at org.apache.hadoop.ipc.Client.call(Client.java:1367) at org.apache.hadoop.ipc.ProtobufRpcEngine$Invoker.invoke(ProtobufRpcEngine.java:228) at org.apache.hadoop.ipc.ProtobufRpcEngine$Invoker.invoke(ProtobufRpcEngine.java:116) at com.sun.proxy.$Proxy45.setTimes(Unknown Source) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.setTimes(ClientNamenodeProtocolTranslatorPB.java:997) at jdk.internal.reflect.GeneratedMethodAccessor41.invoke(Unknown Source) at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(java.base@11.0.14/DelegatingMethodAccessorImpl.java:43) at java.lang.reflect.Method.invoke(java.base@11.0.14/Method.java:566) at org.apache.hadoop.io.retry.RetryInvocationHandler.invokeMethod(RetryInvocationHandler.java:422) at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeMethod(RetryInvocationHandler.java:165) at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invoke(RetryInvocationHandler.java:157) at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeOnce(RetryInvocationHandler.java:95) - locked <0x0000000786e10a50> (a org.apache.hadoop.io.retry.RetryInvocationHandler$Call) at org.apache.hadoop.io.retry.RetryInvocationHandler.invoke(RetryInvocationHandler.java:359) at com.sun.proxy.$Proxy46.setTimes(Unknown Source) at jdk.internal.reflect.GeneratedMethodAccessor41.invoke(Unknown Source) at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(java.base@11.0.14/DelegatingMethodAccessorImpl.java:43) at java.lang.reflect.Method.invoke(java.base@11.0.14/Method.java:566) at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:361) at com.sun.proxy.$Proxy47.setTimes(Unknown Source) at jdk.internal.reflect.GeneratedMethodAccessor41.invoke(Unknown Source) at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(java.base@11.0.14/DelegatingMethodAccessorImpl.java:43) at java.lang.reflect.Method.invoke(java.base@11.0.14/Method.java:566) at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:361) at com.sun.proxy.$Proxy47.setTimes(Unknown Source) at jdk.internal.reflect.GeneratedMethodAccessor41.invoke(Unknown Source) at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(java.base@11.0.14/DelegatingMethodAccessorImpl.java:43) at java.lang.reflect.Method.invoke(java.base@11.0.14/Method.java:566) at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:361) at com.sun.proxy.$Proxy47.setTimes(Unknown Source) at jdk.internal.reflect.GeneratedMethodAccessor41.invoke(Unknown Source) at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(java.base@11.0.14/DelegatingMethodAccessorImpl.java:43) at java.lang.reflect.Method.invoke(java.base@11.0.14/Method.java:566) at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:361) at com.sun.proxy.$Proxy47.setTimes(Unknown Source) at jdk.internal.reflect.GeneratedMethodAccessor41.invoke(Unknown Source) at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(java.base@11.0.14/DelegatingMethodAccessorImpl.java:43) at java.lang.reflect.Method.invoke(java.base@11.0.14/Method.java:566) at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:361) at com.sun.proxy.$Proxy47.setTimes(Unknown Source) at org.apache.hadoop.hdfs.DFSClient.setTimes(DFSClient.java:2552) at org.apache.hadoop.hdfs.DistributedFileSystem$37.doCall(DistributedFileSystem.java:1791) at org.apache.hadoop.hdfs.DistributedFileSystem$37.doCall(DistributedFileSystem.java:1788) at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) at org.apache.hadoop.hdfs.DistributedFileSystem.setTimes(DistributedFileSystem.java:1801) at org.apache.hadoop.hbase.util.CommonFSUtils.renameAndSetModifyTime(CommonFSUtils.java:710) at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.archiveLogFile(AbstractFSWAL.java:765) at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.archive(AbstractFSWAL.java:729) at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$cleanOldLogs$5(AbstractFSWAL.java:718) at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL$$Lambda$674/0x0000000800b9d840.run(Unknown Source) at java.util.concurrent.ThreadPoolExecutor.runWorker(java.base@11.0.14/ThreadPoolExecutor.java:1128) at java.util.concurrent.ThreadPoolExecutor$Worker.run(java.base@11.0.14/ThreadPoolExecutor.java:628) at java.lang.Thread.run(java.base@11.0.14/Thread.java:829) "master/zhangduo-ubuntu:0.Chore.2" #3700 daemon prio=5 os_prio=0 cpu=44.85ms elapsed=6511.15s tid=0x00007fd190024800 nid=0x9b9b waiting on condition [0x00007fd081ef7000] java.lang.Thread.State: WAITING (parking) at jdk.internal.misc.Unsafe.park(java.base@11.0.14/Native Method) - parking to wait for <0x000000077badedc0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.park(java.base@11.0.14/LockSupport.java:194) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(java.base@11.0.14/AbstractQueuedSynchronizer.java:2081) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(java.base@11.0.14/ScheduledThreadPoolExecutor.java:1177) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(java.base@11.0.14/ScheduledThreadPoolExecutor.java:899) at java.util.concurrent.ThreadPoolExecutor.getTask(java.base@11.0.14/ThreadPoolExecutor.java:1054) at java.util.concurrent.ThreadPoolExecutor.runWorker(java.base@11.0.14/ThreadPoolExecutor.java:1114) at java.util.concurrent.ThreadPoolExecutor$Worker.run(java.base@11.0.14/ThreadPoolExecutor.java:628) at java.lang.Thread.run(java.base@11.0.14/Thread.java:829) "region-location-2" #3701 daemon prio=5 os_prio=0 cpu=0.69ms elapsed=6511.14s tid=0x00007fd1480e2800 nid=0x9b9c waiting on condition [0x00007fd093efe000] java.lang.Thread.State: WAITING (parking) at jdk.internal.misc.Unsafe.park(java.base@11.0.14/Native Method) - parking to wait for <0x000000077badf6e0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.park(java.base@11.0.14/LockSupport.java:194) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(java.base@11.0.14/AbstractQueuedSynchronizer.java:2081) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(java.base@11.0.14/ScheduledThreadPoolExecutor.java:1170) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(java.base@11.0.14/ScheduledThreadPoolExecutor.java:899) at java.util.concurrent.ThreadPoolExecutor.getTask(java.base@11.0.14/ThreadPoolExecutor.java:1054) at java.util.concurrent.ThreadPoolExecutor.runWorker(java.base@11.0.14/ThreadPoolExecutor.java:1114) at java.util.concurrent.ThreadPoolExecutor$Worker.run(java.base@11.0.14/ThreadPoolExecutor.java:628) at java.lang.Thread.run(java.base@11.0.14/Thread.java:829) "region-location-3" #3702 daemon prio=5 os_prio=0 cpu=0.51ms elapsed=6511.14s tid=0x00007fd148056000 nid=0x9b9d waiting on condition [0x00007fd11ca10000] java.lang.Thread.State: WAITING (parking) at jdk.internal.misc.Unsafe.park(java.base@11.0.14/Native Method) - parking to wait for <0x000000077badf6e0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.park(java.base@11.0.14/LockSupport.java:194) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(java.base@11.0.14/AbstractQueuedSynchronizer.java:2081) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(java.base@11.0.14/ScheduledThreadPoolExecutor.java:1170) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(java.base@11.0.14/ScheduledThreadPoolExecutor.java:899) at java.util.concurrent.ThreadPoolExecutor.getTask(java.base@11.0.14/ThreadPoolExecutor.java:1054) at java.util.concurrent.ThreadPoolExecutor.runWorker(java.base@11.0.14/ThreadPoolExecutor.java:1114) at java.util.concurrent.ThreadPoolExecutor$Worker.run(java.base@11.0.14/ThreadPoolExecutor.java:628) at java.lang.Thread.run(java.base@11.0.14/Thread.java:829) "region-location-4" #3703 daemon prio=5 os_prio=0 cpu=0.18ms elapsed=6511.14s tid=0x00007fd148057000 nid=0x9b9e waiting on condition [0x00007fd081bf7000] java.lang.Thread.State: WAITING (parking) at jdk.internal.misc.Unsafe.park(java.base@11.0.14/Native Method) - parking to wait for <0x000000077badf6e0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.park(java.base@11.0.14/LockSupport.java:194) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(java.base@11.0.14/AbstractQueuedSynchronizer.java:2081) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(java.base@11.0.14/ScheduledThreadPoolExecutor.java:1170) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(java.base@11.0.14/ScheduledThreadPoolExecutor.java:899) at java.util.concurrent.ThreadPoolExecutor.getTask(java.base@11.0.14/ThreadPoolExecutor.java:1054) at java.util.concurrent.ThreadPoolExecutor.runWorker(java.base@11.0.14/ThreadPoolExecutor.java:1114) at java.util.concurrent.ThreadPoolExecutor$Worker.run(java.base@11.0.14/ThreadPoolExecutor.java:628) at java.lang.Thread.run(java.base@11.0.14/Thread.java:829) "master/zhangduo-ubuntu:0.Chore.3" #4128 daemon prio=5 os_prio=0 cpu=69.20ms elapsed=6495.94s tid=0x00007fd190026000 nid=0x9d4a waiting on condition [0x00007fd07a0f1000] java.lang.Thread.State: TIMED_WAITING (parking) at jdk.internal.misc.Unsafe.park(java.base@11.0.14/Native Method) - parking to wait for <0x000000077badedc0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(java.base@11.0.14/LockSupport.java:234) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(java.base@11.0.14/AbstractQueuedSynchronizer.java:2123) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(java.base@11.0.14/ScheduledThreadPoolExecutor.java:1182) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(java.base@11.0.14/ScheduledThreadPoolExecutor.java:899) at java.util.concurrent.ThreadPoolExecutor.getTask(java.base@11.0.14/ThreadPoolExecutor.java:1054) at java.util.concurrent.ThreadPoolExecutor.runWorker(java.base@11.0.14/ThreadPoolExecutor.java:1114) at java.util.concurrent.ThreadPoolExecutor$Worker.run(java.base@11.0.14/ThreadPoolExecutor.java:628) at java.lang.Thread.run(java.base@11.0.14/Thread.java:829) "master/zhangduo-ubuntu:0.Chore.4" #7438 daemon prio=5 os_prio=0 cpu=32.64ms elapsed=6270.40s tid=0x00007fd190027000 nid=0xab11 waiting on condition [0x00007fd092efe000] java.lang.Thread.State: TIMED_WAITING (parking) at jdk.internal.misc.Unsafe.park(java.base@11.0.14/Native Method) - parking to wait for <0x000000077badedc0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(java.base@11.0.14/LockSupport.java:234) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(java.base@11.0.14/AbstractQueuedSynchronizer.java:2123) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(java.base@11.0.14/ScheduledThreadPoolExecutor.java:1182) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(java.base@11.0.14/ScheduledThreadPoolExecutor.java:899) at java.util.concurrent.ThreadPoolExecutor.getTask(java.base@11.0.14/ThreadPoolExecutor.java:1054) at java.util.concurrent.ThreadPoolExecutor.runWorker(java.base@11.0.14/ThreadPoolExecutor.java:1114) at java.util.concurrent.ThreadPoolExecutor$Worker.run(java.base@11.0.14/ThreadPoolExecutor.java:628) at java.lang.Thread.run(java.base@11.0.14/Thread.java:829) "master/zhangduo-ubuntu:0.Chore.5" #7439 daemon prio=5 os_prio=0 cpu=93.65ms elapsed=6270.40s tid=0x00007fd0ac035800 nid=0xab12 waiting on condition [0x00007fd093cfd000] java.lang.Thread.State: TIMED_WAITING (sleeping) at java.lang.Thread.sleep(java.base@11.0.14/Native Method) at org.apache.hadoop.hbase.client.ConnectionImplementation.locateRegionInMeta(ConnectionImplementation.java:1055) at org.apache.hadoop.hbase.client.ConnectionImplementation.locateRegion(ConnectionImplementation.java:849) at org.apache.hadoop.hbase.client.RpcRetryingCallerWithReadReplicas.getRegionLocations(RpcRetryingCallerWithReadReplicas.java:319) at org.apache.hadoop.hbase.client.ScannerCallableWithReplicas.call(ScannerCallableWithReplicas.java:157) at org.apache.hadoop.hbase.client.ScannerCallableWithReplicas.call(ScannerCallableWithReplicas.java:53) at org.apache.hadoop.hbase.client.RpcRetryingCallerImpl.callWithoutRetries(RpcRetryingCallerImpl.java:192) at org.apache.hadoop.hbase.client.ClientScanner.call(ClientScanner.java:270) at org.apache.hadoop.hbase.client.ClientScanner.loadCache(ClientScanner.java:439) at org.apache.hadoop.hbase.client.ClientScanner.nextWithSyncCache(ClientScanner.java:313) at org.apache.hadoop.hbase.client.ClientScanner.next(ClientScanner.java:611) at org.apache.hadoop.hbase.client.ResultScanner$1.hasNext(ResultScanner.java:51) at org.apache.hadoop.hbase.quotas.QuotaTableUtil.getTableSnapshots(QuotaTableUtil.java:808) at org.apache.hadoop.hbase.quotas.SnapshotQuotaObserverChore.pruneTableSnapshots(SnapshotQuotaObserverChore.java:130) at org.apache.hadoop.hbase.quotas.SnapshotQuotaObserverChore._chore(SnapshotQuotaObserverChore.java:112) at org.apache.hadoop.hbase.quotas.SnapshotQuotaObserverChore.chore(SnapshotQuotaObserverChore.java:93) at org.apache.hadoop.hbase.ScheduledChore.run(ScheduledChore.java:159) at java.util.concurrent.Executors$RunnableAdapter.call(java.base@11.0.14/Executors.java:515) at java.util.concurrent.FutureTask.runAndReset(java.base@11.0.14/FutureTask.java:305) at java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.run(java.base@11.0.14/ScheduledThreadPoolExecutor.java:305) at org.apache.hadoop.hbase.JitterScheduledThreadPoolExecutorImpl$JitteredRunnableScheduledFuture.run(JitterScheduledThreadPoolExecutorImpl.java:107) at java.util.concurrent.ThreadPoolExecutor.runWorker(java.base@11.0.14/ThreadPoolExecutor.java:1128) at java.util.concurrent.ThreadPoolExecutor$Worker.run(java.base@11.0.14/ThreadPoolExecutor.java:628) at java.lang.Thread.run(java.base@11.0.14/Thread.java:829) "master/zhangduo-ubuntu:0.Chore.6" #7440 daemon prio=5 os_prio=0 cpu=25.93ms elapsed=6270.40s tid=0x00007fd190034000 nid=0xab13 waiting on condition [0x00007fd0984fe000] java.lang.Thread.State: TIMED_WAITING (parking) at jdk.internal.misc.Unsafe.park(java.base@11.0.14/Native Method) - parking to wait for <0x000000077badedc0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(java.base@11.0.14/LockSupport.java:234) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(java.base@11.0.14/AbstractQueuedSynchronizer.java:2123) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(java.base@11.0.14/ScheduledThreadPoolExecutor.java:1182) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(java.base@11.0.14/ScheduledThreadPoolExecutor.java:899) at java.util.concurrent.ThreadPoolExecutor.getTask(java.base@11.0.14/ThreadPoolExecutor.java:1054) at java.util.concurrent.ThreadPoolExecutor.runWorker(java.base@11.0.14/ThreadPoolExecutor.java:1114) at java.util.concurrent.ThreadPoolExecutor$Worker.run(java.base@11.0.14/ThreadPoolExecutor.java:628) at java.lang.Thread.run(java.base@11.0.14/Thread.java:829) "master/zhangduo-ubuntu:0.Chore.7" #10093 daemon prio=5 os_prio=0 cpu=22.49ms elapsed=5970.39s tid=0x00007fd190035800 nid=0xb589 waiting on condition [0x00007fd093dfe000] java.lang.Thread.State: WAITING (parking) at jdk.internal.misc.Unsafe.park(java.base@11.0.14/Native Method) - parking to wait for <0x000000077badedc0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.park(java.base@11.0.14/LockSupport.java:194) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(java.base@11.0.14/AbstractQueuedSynchronizer.java:2081) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(java.base@11.0.14/ScheduledThreadPoolExecutor.java:1177) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(java.base@11.0.14/ScheduledThreadPoolExecutor.java:899) at java.util.concurrent.ThreadPoolExecutor.getTask(java.base@11.0.14/ThreadPoolExecutor.java:1054) at java.util.concurrent.ThreadPoolExecutor.runWorker(java.base@11.0.14/ThreadPoolExecutor.java:1114) at java.util.concurrent.ThreadPoolExecutor$Worker.run(java.base@11.0.14/ThreadPoolExecutor.java:628) at java.lang.Thread.run(java.base@11.0.14/Thread.java:829) "master/zhangduo-ubuntu:0.Chore.8" #10095 daemon prio=5 os_prio=0 cpu=23.43ms elapsed=5970.39s tid=0x00007fd14806d800 nid=0xb58a waiting on condition [0x00007fd08d5fe000] java.lang.Thread.State: WAITING (parking) at jdk.internal.misc.Unsafe.park(java.base@11.0.14/Native Method) - parking to wait for <0x000000077badedc0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.park(java.base@11.0.14/LockSupport.java:194) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(java.base@11.0.14/AbstractQueuedSynchronizer.java:2081) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(java.base@11.0.14/ScheduledThreadPoolExecutor.java:1177) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(java.base@11.0.14/ScheduledThreadPoolExecutor.java:899) at java.util.concurrent.ThreadPoolExecutor.getTask(java.base@11.0.14/ThreadPoolExecutor.java:1054) at java.util.concurrent.ThreadPoolExecutor.runWorker(java.base@11.0.14/ThreadPoolExecutor.java:1114) at java.util.concurrent.ThreadPoolExecutor$Worker.run(java.base@11.0.14/ThreadPoolExecutor.java:628) at java.lang.Thread.run(java.base@11.0.14/Thread.java:829) "surefire-forkedjvm-last-ditch-daemon-shutdown-thread-30s" #11289 daemon prio=5 os_prio=0 cpu=8.00ms elapsed=5798.35s tid=0x00007fd1b9163000 nid=0xba43 waiting on condition [0x00007fd09aefe000] java.lang.Thread.State: WAITING (parking) at jdk.internal.misc.Unsafe.park(java.base@11.0.14/Native Method) - parking to wait for <0x0000000780ea28d0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.park(java.base@11.0.14/LockSupport.java:194) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(java.base@11.0.14/AbstractQueuedSynchronizer.java:2081) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(java.base@11.0.14/ScheduledThreadPoolExecutor.java:1170) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(java.base@11.0.14/ScheduledThreadPoolExecutor.java:899) at java.util.concurrent.ThreadPoolExecutor.getTask(java.base@11.0.14/ThreadPoolExecutor.java:1054) at java.util.concurrent.ThreadPoolExecutor.runWorker(java.base@11.0.14/ThreadPoolExecutor.java:1114) at java.util.concurrent.ThreadPoolExecutor$Worker.run(java.base@11.0.14/ThreadPoolExecutor.java:628) at java.lang.Thread.run(java.base@11.0.14/Thread.java:829) "DestroyJavaVM" #11358 prio=5 os_prio=0 cpu=1077.63ms elapsed=5788.03s tid=0x00007fd1b8027800 nid=0x8a9b waiting on condition [0x0000000000000000] java.lang.Thread.State: RUNNABLE "IPC Parameter Sending Thread #10" #14836 daemon prio=5 os_prio=0 cpu=683997.47ms elapsed=5156.75s tid=0x00007fd12c18e000 nid=0xc844 waiting on condition [0x00007fd0818f7000] java.lang.Thread.State: TIMED_WAITING (parking) at jdk.internal.misc.Unsafe.park(java.base@11.0.14/Native Method) - parking to wait for <0x000000077b1bd5e0> (a java.util.concurrent.SynchronousQueue$TransferStack) at java.util.concurrent.locks.LockSupport.parkNanos(java.base@11.0.14/LockSupport.java:234) at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(java.base@11.0.14/SynchronousQueue.java:462) at java.util.concurrent.SynchronousQueue$TransferStack.transfer(java.base@11.0.14/SynchronousQueue.java:361) at java.util.concurrent.SynchronousQueue.poll(java.base@11.0.14/SynchronousQueue.java:937) at java.util.concurrent.ThreadPoolExecutor.getTask(java.base@11.0.14/ThreadPoolExecutor.java:1053) at java.util.concurrent.ThreadPoolExecutor.runWorker(java.base@11.0.14/ThreadPoolExecutor.java:1114) at java.util.concurrent.ThreadPoolExecutor$Worker.run(java.base@11.0.14/ThreadPoolExecutor.java:628) at java.lang.Thread.run(java.base@11.0.14/Thread.java:829) "java.util.concurrent.ThreadPoolExecutor$Worker@313c176f[State = -1, empty queue]" #15959 daemon prio=5 os_prio=0 cpu=235.56ms elapsed=4910.67s tid=0x00007fd12462f800 nid=0xccac waiting on condition [0x00007fd082ff7000] java.lang.Thread.State: WAITING (parking) at jdk.internal.misc.Unsafe.park(java.base@11.0.14/Native Method) - parking to wait for <0x000000077b210e70> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.park(java.base@11.0.14/LockSupport.java:194) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(java.base@11.0.14/AbstractQueuedSynchronizer.java:2081) at java.util.concurrent.LinkedBlockingQueue.take(java.base@11.0.14/LinkedBlockingQueue.java:433) at java.util.concurrent.ThreadPoolExecutor.getTask(java.base@11.0.14/ThreadPoolExecutor.java:1054) at java.util.concurrent.ThreadPoolExecutor.runWorker(java.base@11.0.14/ThreadPoolExecutor.java:1114) at java.util.concurrent.ThreadPoolExecutor$Worker.run(java.base@11.0.14/ThreadPoolExecutor.java:628) at java.lang.Thread.run(java.base@11.0.14/Thread.java:829) "master/zhangduo-ubuntu:0.Chore.9" #16574 daemon prio=5 os_prio=0 cpu=33.64ms elapsed=4768.31s tid=0x00007fd14806e800 nid=0xcf1e waiting on condition [0x00007fd073df4000] java.lang.Thread.State: WAITING (parking) at jdk.internal.misc.Unsafe.park(java.base@11.0.14/Native Method) - parking to wait for <0x000000077badedc0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.park(java.base@11.0.14/LockSupport.java:194) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(java.base@11.0.14/AbstractQueuedSynchronizer.java:2081) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(java.base@11.0.14/ScheduledThreadPoolExecutor.java:1177) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(java.base@11.0.14/ScheduledThreadPoolExecutor.java:899) at java.util.concurrent.ThreadPoolExecutor.getTask(java.base@11.0.14/ThreadPoolExecutor.java:1054) at java.util.concurrent.ThreadPoolExecutor.runWorker(java.base@11.0.14/ThreadPoolExecutor.java:1114) at java.util.concurrent.ThreadPoolExecutor$Worker.run(java.base@11.0.14/ThreadPoolExecutor.java:628) at java.lang.Thread.run(java.base@11.0.14/Thread.java:829) "IPC Parameter Sending Thread #12" #17217 daemon prio=5 os_prio=0 cpu=506525.71ms elapsed=4615.42s tid=0x00007fd0f00d1000 nid=0xd1a9 waiting on condition [0x00007fd08c8fe000] java.lang.Thread.State: TIMED_WAITING (parking) at jdk.internal.misc.Unsafe.park(java.base@11.0.14/Native Method) - parking to wait for <0x000000077b1bd5e0> (a java.util.concurrent.SynchronousQueue$TransferStack) at java.util.concurrent.locks.LockSupport.parkNanos(java.base@11.0.14/LockSupport.java:234) at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(java.base@11.0.14/SynchronousQueue.java:462) at java.util.concurrent.SynchronousQueue$TransferStack.transfer(java.base@11.0.14/SynchronousQueue.java:361) at java.util.concurrent.SynchronousQueue.poll(java.base@11.0.14/SynchronousQueue.java:937) at java.util.concurrent.ThreadPoolExecutor.getTask(java.base@11.0.14/ThreadPoolExecutor.java:1053) at java.util.concurrent.ThreadPoolExecutor.runWorker(java.base@11.0.14/ThreadPoolExecutor.java:1114) at java.util.concurrent.ThreadPoolExecutor$Worker.run(java.base@11.0.14/ThreadPoolExecutor.java:628) at java.lang.Thread.run(java.base@11.0.14/Thread.java:829) "Timer for 'HBase' metrics system" #17623 daemon prio=5 os_prio=0 cpu=1.42ms elapsed=266.46s tid=0x00007fd16801b800 nid=0xe558 in Object.wait() [0x00007fd07dcf4000] java.lang.Thread.State: TIMED_WAITING (on object monitor) at java.lang.Object.wait(java.base@11.0.14/Native Method) - waiting on at java.util.TimerThread.mainLoop(java.base@11.0.14/Timer.java:553) - waiting to re-lock in wait() <0x00000007805e2978> (a java.util.TaskQueue) at java.util.TimerThread.run(java.base@11.0.14/Timer.java:506) "DataXceiver for client DFSClient_NONMAPREDUCE_-1834119122_18 at /127.0.0.1:52598 [Receiving block BP-1865003422-127.0.0.1-1652709596755:blk_1073746001_5177]" #17630 daemon prio=5 os_prio=0 cpu=0.82ms elapsed=77.66s tid=0x00007fd0f800a800 nid=0xe72f runnable [0x00007fd08f4fd000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPoll.wait(java.base@11.0.14/Native Method) at sun.nio.ch.EPollSelectorImpl.doSelect(java.base@11.0.14/EPollSelectorImpl.java:120) at sun.nio.ch.SelectorImpl.lockAndDoSelect(java.base@11.0.14/SelectorImpl.java:124) - locked <0x000000078b533138> (a sun.nio.ch.Util$2) - locked <0x000000078b5330c0> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(java.base@11.0.14/SelectorImpl.java:136) at org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) at org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:157) at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) at java.io.BufferedInputStream.fill(java.base@11.0.14/BufferedInputStream.java:252) at java.io.BufferedInputStream.read1(java.base@11.0.14/BufferedInputStream.java:292) at java.io.BufferedInputStream.read(java.base@11.0.14/BufferedInputStream.java:351) - locked <0x000000078200f328> (a java.io.BufferedInputStream) at java.io.DataInputStream.read(java.base@11.0.14/DataInputStream.java:149) at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:210) at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:211) at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:134) at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:109) at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:528) at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:971) at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:891) at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:173) at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:107) at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:290) at java.lang.Thread.run(java.base@11.0.14/Thread.java:829) "DataXceiver for client DFSClient_NONMAPREDUCE_-1834119122_18 at /127.0.0.1:50258 [Receiving block BP-1865003422-127.0.0.1-1652709596755:blk_1073746001_5177]" #17631 daemon prio=5 os_prio=0 cpu=0.60ms elapsed=77.66s tid=0x00007fd16803b000 nid=0xe730 runnable [0x00007fd09a5fd000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPoll.wait(java.base@11.0.14/Native Method) at sun.nio.ch.EPollSelectorImpl.doSelect(java.base@11.0.14/EPollSelectorImpl.java:120) at sun.nio.ch.SelectorImpl.lockAndDoSelect(java.base@11.0.14/SelectorImpl.java:124) - locked <0x000000077daa4cb0> (a sun.nio.ch.Util$2) - locked <0x000000077daa4ba8> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(java.base@11.0.14/SelectorImpl.java:136) at org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) at org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:157) at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) at java.io.BufferedInputStream.fill(java.base@11.0.14/BufferedInputStream.java:252) at java.io.BufferedInputStream.read1(java.base@11.0.14/BufferedInputStream.java:292) at java.io.BufferedInputStream.read(java.base@11.0.14/BufferedInputStream.java:351) - locked <0x0000000782011240> (a java.io.BufferedInputStream) at java.io.DataInputStream.read(java.base@11.0.14/DataInputStream.java:149) at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:210) at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:211) at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:134) at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:109) at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:528) at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:971) at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:891) at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:173) at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:107) at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:290) at java.lang.Thread.run(java.base@11.0.14/Thread.java:829) "PacketResponder: BP-1865003422-127.0.0.1-1652709596755:blk_1073746001_5177, type=LAST_IN_PIPELINE" #17632 daemon prio=5 os_prio=0 cpu=0.46ms elapsed=77.65s tid=0x00007fd0fc03e800 nid=0xe731 in Object.wait() [0x00007fd0746ee000] java.lang.Thread.State: WAITING (on object monitor) at java.lang.Object.wait(java.base@11.0.14/Native Method) - waiting on at java.lang.Object.wait(java.base@11.0.14/Object.java:328) at org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.waitForAckHead(BlockReceiver.java:1330) - waiting to re-lock in wait() <0x0000000782010270> (a java.util.LinkedList) at org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1402) at java.lang.Thread.run(java.base@11.0.14/Thread.java:829) "PacketResponder: BP-1865003422-127.0.0.1-1652709596755:blk_1073746001_5177, type=LAST_IN_PIPELINE" #17633 daemon prio=5 os_prio=0 cpu=0.64ms elapsed=77.65s tid=0x00007fd108006000 nid=0xe732 in Object.wait() [0x00007fd174ff0000] java.lang.Thread.State: WAITING (on object monitor) at java.lang.Object.wait(java.base@11.0.14/Native Method) - waiting on at java.lang.Object.wait(java.base@11.0.14/Object.java:328) at org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.waitForAckHead(BlockReceiver.java:1330) - waiting to re-lock in wait() <0x000000078200c598> (a java.util.LinkedList) at org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1402) at java.lang.Thread.run(java.base@11.0.14/Thread.java:829) "hconnection-0x669e0b57-metaLookup-shared--pool-113" #17636 prio=5 os_prio=0 cpu=4.77ms elapsed=31.07s tid=0x00007fd190036000 nid=0xe78c in Object.wait() [0x00007fd073bf4000] java.lang.Thread.State: TIMED_WAITING (on object monitor) at java.lang.Object.wait(java.base@11.0.14/Native Method) - waiting on at org.apache.hadoop.hbase.client.RpcRetryingCallerImpl.callWithRetries(RpcRetryingCallerImpl.java:168) - waiting to re-lock in wait() <0x00000007824b2610> (a java.util.concurrent.atomic.AtomicBoolean) at org.apache.hadoop.hbase.client.ResultBoundedCompletionService$QueueingFuture.run(ResultBoundedCompletionService.java:73) at java.util.concurrent.ThreadPoolExecutor.runWorker(java.base@11.0.14/ThreadPoolExecutor.java:1128) at java.util.concurrent.ThreadPoolExecutor$Worker.run(java.base@11.0.14/ThreadPoolExecutor.java:628) at java.lang.Thread.run(java.base@11.0.14/Thread.java:829) "Attach Listener" #17637 daemon prio=9 os_prio=0 cpu=0.84ms elapsed=22.80s tid=0x00007fd170010000 nid=0xe7a0 waiting on condition [0x0000000000000000] java.lang.Thread.State: RUNNABLE "VM Thread" os_prio=0 cpu=9424.50ms elapsed=6579.51s tid=0x00007fd1b829f000 nid=0x8aa4 runnable "GC Thread#0" os_prio=0 cpu=11359.70ms elapsed=6579.53s tid=0x00007fd1b8041000 nid=0x8a9c runnable "GC Thread#1" os_prio=0 cpu=11319.67ms elapsed=6579.19s tid=0x00007fd184001000 nid=0x8ab1 runnable "GC Thread#2" os_prio=0 cpu=11342.82ms elapsed=6579.19s tid=0x00007fd184002800 nid=0x8ab2 runnable "GC Thread#3" os_prio=0 cpu=11345.41ms elapsed=6579.19s tid=0x00007fd184004800 nid=0x8ab3 runnable "GC Thread#4" os_prio=0 cpu=11380.52ms elapsed=6579.19s tid=0x00007fd184006000 nid=0x8ab4 runnable "GC Thread#5" os_prio=0 cpu=11459.50ms elapsed=6579.19s tid=0x00007fd184008000 nid=0x8ab5 runnable "GC Thread#6" os_prio=0 cpu=11399.85ms elapsed=6579.18s tid=0x00007fd18400a000 nid=0x8ab6 runnable "GC Thread#7" os_prio=0 cpu=11308.06ms elapsed=6579.18s tid=0x00007fd18400b800 nid=0x8ab7 runnable "G1 Main Marker" os_prio=0 cpu=8.02ms elapsed=6579.53s tid=0x00007fd1b8077800 nid=0x8a9d runnable "G1 Conc#0" os_prio=0 cpu=361.55ms elapsed=6579.52s tid=0x00007fd1b8079800 nid=0x8a9e runnable "G1 Conc#1" os_prio=0 cpu=364.64ms elapsed=6578.49s tid=0x00007fd190001000 nid=0x8abc runnable "G1 Refine#0" os_prio=0 cpu=6.35ms elapsed=6579.52s tid=0x00007fd1b81d4000 nid=0x8a9f runnable "G1 Young RemSet Sampling" os_prio=0 cpu=1273.39ms elapsed=6579.52s tid=0x00007fd1b81d6000 nid=0x8aa0 runnable "VM Periodic Task Thread" os_prio=0 cpu=3873.19ms elapsed=6579.45s tid=0x00007fd1b83e2000 nid=0x8aae waiting on condition JNI global refs: 32, weak refs: 0