2022-08-26 22:42:13 Full thread dump OpenJDK 64-Bit Server VM (11.0.15+10 mixed mode): Threads class SMR info: _java_thread_list=0x00007fbe0c001680, length=19, elements={ 0x00007fbe2829a000, 0x00007fbe2829e000, 0x00007fbe282b3000, 0x00007fbe282b5000, 0x00007fbe282b7800, 0x00007fbe282b9800, 0x00007fbe282bb800, 0x00007fbe2836f800, 0x00007fbe29a49000, 0x00007fbe2a23b800, 0x00007fbe2a277800, 0x00007fbe2a279000, 0x00007fbe2a36c800, 0x00007fbe2a437000, 0x00007fbe2a4a1800, 0x00007fbe0c455800, 0x00007fbe29549000, 0x00007fbe2802c000, 0x00007fbe0c2b7800 } "Reference Handler" #2 daemon prio=10 os_prio=0 cpu=3.37ms elapsed=38.80s tid=0x00007fbe2829a000 nid=0x1b09 waiting on condition [0x00007fbe10416000] java.lang.Thread.State: RUNNABLE at java.lang.ref.Reference.waitForReferencePendingList(java.base@11.0.15/Native Method) at java.lang.ref.Reference.processPendingReferences(java.base@11.0.15/Reference.java:241) at java.lang.ref.Reference$ReferenceHandler.run(java.base@11.0.15/Reference.java:213) "Finalizer" #3 daemon prio=8 os_prio=0 cpu=1.80ms elapsed=38.80s tid=0x00007fbe2829e000 nid=0x1b0a in Object.wait() [0x00007fbe10316000] java.lang.Thread.State: WAITING (on object monitor) at java.lang.Object.wait(java.base@11.0.15/Native Method) - waiting on <0x0000000689a581a0> (a java.lang.ref.ReferenceQueue$Lock) at java.lang.ref.ReferenceQueue.remove(java.base@11.0.15/ReferenceQueue.java:155) - waiting to re-lock in wait() <0x0000000689a581a0> (a java.lang.ref.ReferenceQueue$Lock) at java.lang.ref.ReferenceQueue.remove(java.base@11.0.15/ReferenceQueue.java:176) at java.lang.ref.Finalizer$FinalizerThread.run(java.base@11.0.15/Finalizer.java:170) "Signal Dispatcher" #4 daemon prio=9 os_prio=0 cpu=0.31ms elapsed=38.80s tid=0x00007fbe282b3000 nid=0x1b0b runnable [0x0000000000000000] java.lang.Thread.State: RUNNABLE "Service Thread" #5 daemon prio=9 os_prio=0 cpu=0.18ms elapsed=38.80s tid=0x00007fbe282b5000 nid=0x1b0c runnable [0x0000000000000000] java.lang.Thread.State: RUNNABLE "C2 CompilerThread0" #6 daemon prio=9 os_prio=0 cpu=3719.38ms elapsed=38.80s tid=0x00007fbe282b7800 nid=0x1b0d waiting on condition [0x0000000000000000] java.lang.Thread.State: RUNNABLE No compile task "C1 CompilerThread0" #9 daemon prio=9 os_prio=0 cpu=1892.14ms elapsed=38.80s tid=0x00007fbe282b9800 nid=0x1b0e waiting on condition [0x0000000000000000] java.lang.Thread.State: RUNNABLE No compile task "Sweeper thread" #10 daemon prio=9 os_prio=0 cpu=4.28ms elapsed=38.80s tid=0x00007fbe282bb800 nid=0x1b0f runnable [0x0000000000000000] java.lang.Thread.State: RUNNABLE "Common-Cleaner" #11 daemon prio=8 os_prio=0 cpu=6.49ms elapsed=38.77s tid=0x00007fbe2836f800 nid=0x1b10 in Object.wait() [0x00007fbdf6d86000] java.lang.Thread.State: TIMED_WAITING (on object monitor) at java.lang.Object.wait(java.base@11.0.15/Native Method) - waiting on <0x0000000689a58c50> (a java.lang.ref.ReferenceQueue$Lock) at java.lang.ref.ReferenceQueue.remove(java.base@11.0.15/ReferenceQueue.java:155) - waiting to re-lock in wait() <0x0000000689a58c50> (a java.lang.ref.ReferenceQueue$Lock) at jdk.internal.ref.CleanerImpl.run(java.base@11.0.15/CleanerImpl.java:148) at java.lang.Thread.run(java.base@11.0.15/Thread.java:829) at jdk.internal.misc.InnocuousThread.run(java.base@11.0.15/InnocuousThread.java:161) "process reaper" #16 daemon prio=10 os_prio=0 cpu=20.61ms elapsed=36.76s tid=0x00007fbe29a49000 nid=0x1b1d waiting on condition [0x00007fbe2d85b000] java.lang.Thread.State: TIMED_WAITING (parking) at jdk.internal.misc.Unsafe.park(java.base@11.0.15/Native Method) - parking to wait for <0x00000006ae9c6720> (a java.util.concurrent.SynchronousQueue$TransferStack) at java.util.concurrent.locks.LockSupport.parkNanos(java.base@11.0.15/LockSupport.java:234) at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(java.base@11.0.15/SynchronousQueue.java:462) at java.util.concurrent.SynchronousQueue$TransferStack.transfer(java.base@11.0.15/SynchronousQueue.java:361) at java.util.concurrent.SynchronousQueue.poll(java.base@11.0.15/SynchronousQueue.java:937) at java.util.concurrent.ThreadPoolExecutor.getTask(java.base@11.0.15/ThreadPoolExecutor.java:1053) at java.util.concurrent.ThreadPoolExecutor.runWorker(java.base@11.0.15/ThreadPoolExecutor.java:1114) at java.util.concurrent.ThreadPoolExecutor$Worker.run(java.base@11.0.15/ThreadPoolExecutor.java:628) at java.lang.Thread.run(java.base@11.0.15/Thread.java:829) "ReadOnlyZKClient-127.0.0.1:2181@0x4714f447" #21 daemon prio=5 os_prio=0 cpu=84.35ms elapsed=35.11s tid=0x00007fbe2a23b800 nid=0x1b27 waiting on condition [0x00007fbdf25fe000] java.lang.Thread.State: TIMED_WAITING (parking) at jdk.internal.misc.Unsafe.park(java.base@11.0.15/Native Method) - parking to wait for <0x00000006ae800198> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(java.base@11.0.15/LockSupport.java:234) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(java.base@11.0.15/AbstractQueuedSynchronizer.java:2123) at java.util.concurrent.DelayQueue.poll(java.base@11.0.15/DelayQueue.java:265) at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:328) at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$$Lambda$311/0x00000008007fec40.run(Unknown Source) at java.lang.Thread.run(java.base@11.0.15/Thread.java:829) "ReadOnlyZKClient-127.0.0.1:2181@0x4714f447-SendThread(127.0.0.1:2181)" #22 daemon prio=5 os_prio=0 cpu=19.86ms elapsed=35.07s tid=0x00007fbe2a277800 nid=0x1b28 runnable [0x00007fbdf24fe000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPoll.wait(java.base@11.0.15/Native Method) at sun.nio.ch.EPollSelectorImpl.doSelect(java.base@11.0.15/EPollSelectorImpl.java:120) at sun.nio.ch.SelectorImpl.lockAndDoSelect(java.base@11.0.15/SelectorImpl.java:124) - locked <0x00000006ae8c69a8> (a sun.nio.ch.Util$2) - locked <0x00000006ae8c6950> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(java.base@11.0.15/SelectorImpl.java:136) at org.apache.zookeeper.ClientCnxnSocketNIO.doTransport(ClientCnxnSocketNIO.java:345) at org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1223) "ReadOnlyZKClient-127.0.0.1:2181@0x4714f447-EventThread" #23 daemon prio=5 os_prio=0 cpu=2.25ms elapsed=35.07s tid=0x00007fbe2a279000 nid=0x1b29 waiting on condition [0x00007fbdf23fe000] java.lang.Thread.State: WAITING (parking) at jdk.internal.misc.Unsafe.park(java.base@11.0.15/Native Method) - parking to wait for <0x00000006ae9c69e8> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.park(java.base@11.0.15/LockSupport.java:194) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(java.base@11.0.15/AbstractQueuedSynchronizer.java:2081) at java.util.concurrent.LinkedBlockingQueue.take(java.base@11.0.15/LinkedBlockingQueue.java:433) at org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:506) "Idle-Rpc-Conn-Sweeper-pool-0" #25 daemon prio=5 os_prio=0 cpu=0.26ms elapsed=34.91s tid=0x00007fbe2a36c800 nid=0x1b2a waiting on condition [0x00007fbdf67fe000] java.lang.Thread.State: TIMED_WAITING (parking) at jdk.internal.misc.Unsafe.park(java.base@11.0.15/Native Method) - parking to wait for <0x00000006ae863608> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(java.base@11.0.15/LockSupport.java:234) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(java.base@11.0.15/AbstractQueuedSynchronizer.java:2123) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(java.base@11.0.15/ScheduledThreadPoolExecutor.java:1182) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(java.base@11.0.15/ScheduledThreadPoolExecutor.java:899) at java.util.concurrent.ThreadPoolExecutor.getTask(java.base@11.0.15/ThreadPoolExecutor.java:1054) at java.util.concurrent.ThreadPoolExecutor.runWorker(java.base@11.0.15/ThreadPoolExecutor.java:1114) at java.util.concurrent.ThreadPoolExecutor$Worker.run(java.base@11.0.15/ThreadPoolExecutor.java:628) at java.lang.Thread.run(java.base@11.0.15/Thread.java:829) "jnr.ffi.util.ref.internal.Finalizer" #26 daemon prio=10 os_prio=0 cpu=0.13ms elapsed=34.46s tid=0x00007fbe2a437000 nid=0x1b30 in Object.wait() [0x00007fbdf1afe000] java.lang.Thread.State: WAITING (on object monitor) at java.lang.Object.wait(java.base@11.0.15/Native Method) - waiting on <0x00000006a3098a70> (a java.lang.ref.ReferenceQueue$Lock) at java.lang.ref.ReferenceQueue.remove(java.base@11.0.15/ReferenceQueue.java:155) - waiting to re-lock in wait() <0x00000006a3098a70> (a java.lang.ref.ReferenceQueue$Lock) at java.lang.ref.ReferenceQueue.remove(java.base@11.0.15/ReferenceQueue.java:176) at jnr.ffi.util.ref.internal.Finalizer.run(Finalizer.java:180) at java.lang.Thread.run(java.base@11.0.15/Thread.java:829) "hconnection-0x656672fb-shared-pool-0" #27 prio=5 os_prio=0 cpu=179.83ms elapsed=29.72s tid=0x00007fbe2a4a1800 nid=0x1b3b waiting on condition [0x00007fbdf1dfe000] java.lang.Thread.State: TIMED_WAITING (parking) at jdk.internal.misc.Unsafe.park(java.base@11.0.15/Native Method) - parking to wait for <0x00000006a2a50e08> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(java.base@11.0.15/LockSupport.java:234) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(java.base@11.0.15/AbstractQueuedSynchronizer.java:2123) at java.util.concurrent.LinkedBlockingQueue.poll(java.base@11.0.15/LinkedBlockingQueue.java:458) at java.util.concurrent.ThreadPoolExecutor.getTask(java.base@11.0.15/ThreadPoolExecutor.java:1053) at java.util.concurrent.ThreadPoolExecutor.runWorker(java.base@11.0.15/ThreadPoolExecutor.java:1114) at java.util.concurrent.ThreadPoolExecutor$Worker.run(java.base@11.0.15/ThreadPoolExecutor.java:628) at java.lang.Thread.run(java.base@11.0.15/Thread.java:829) "RPCClient-NioEventLoopGroup-1-1" #28 daemon prio=5 os_prio=0 cpu=80.72ms elapsed=29.54s tid=0x00007fbe0c455800 nid=0x1b3c runnable [0x00007fbdf15fe000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPoll.wait(java.base@11.0.15/Native Method) at sun.nio.ch.EPollSelectorImpl.doSelect(java.base@11.0.15/EPollSelectorImpl.java:120) at sun.nio.ch.SelectorImpl.lockAndDoSelect(java.base@11.0.15/SelectorImpl.java:124) - locked <0x00000006ae6cae28> (a org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySet) - locked <0x00000006ae6cbe50> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(java.base@11.0.15/SelectorImpl.java:136) at org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:62) at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:817) at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:460) at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) at java.lang.Thread.run(java.base@11.0.15/Thread.java:829) "RpcClient-timer-pool-0" #24 daemon prio=5 os_prio=0 cpu=124.71ms elapsed=29.49s tid=0x00007fbe29549000 nid=0x1b3d waiting on condition [0x00007fbdf14fe000] java.lang.Thread.State: TIMED_WAITING (sleeping) at java.lang.Thread.sleep(java.base@11.0.15/Native Method) at org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$Worker.waitForNextTick(HashedWheelTimer.java:600) at org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$Worker.run(HashedWheelTimer.java:496) at java.lang.Thread.run(java.base@11.0.15/Thread.java:829) "DestroyJavaVM" #29 prio=5 os_prio=0 cpu=4169.67ms elapsed=27.84s tid=0x00007fbe2802c000 nid=0x1b02 waiting on condition [0x0000000000000000] java.lang.Thread.State: RUNNABLE "Attach Listener" #30 daemon prio=9 os_prio=0 cpu=0.37ms elapsed=0.10s tid=0x00007fbe0c2b7800 nid=0x1b62 waiting on condition [0x0000000000000000] java.lang.Thread.State: RUNNABLE "VM Thread" os_prio=0 cpu=160.01ms elapsed=38.80s tid=0x00007fbe28291800 nid=0x1b08 runnable "GC Thread#0" os_prio=0 cpu=94.66ms elapsed=38.81s tid=0x00007fbe28045800 nid=0x1b03 runnable "GC Thread#1" os_prio=0 cpu=88.38ms elapsed=38.56s tid=0x00007fbe28eb3800 nid=0x1b14 runnable "GC Thread#2" os_prio=0 cpu=83.05ms elapsed=38.56s tid=0x00007fbe28eb8000 nid=0x1b15 runnable "GC Thread#3" os_prio=0 cpu=76.12ms elapsed=38.56s tid=0x00007fbe28eb9800 nid=0x1b16 runnable "GC Thread#4" os_prio=0 cpu=83.38ms elapsed=38.56s tid=0x00007fbe28eba800 nid=0x1b17 runnable "GC Thread#5" os_prio=0 cpu=90.05ms elapsed=38.56s tid=0x00007fbe28ebc800 nid=0x1b18 runnable "GC Thread#6" os_prio=0 cpu=71.53ms elapsed=38.56s tid=0x00007fbe28ebe000 nid=0x1b19 runnable "GC Thread#7" os_prio=0 cpu=90.08ms elapsed=38.56s tid=0x00007fbe28ebf800 nid=0x1b1a runnable "G1 Main Marker" os_prio=0 cpu=2.63ms elapsed=38.81s tid=0x00007fbe280a1800 nid=0x1b04 runnable "G1 Conc#0" os_prio=0 cpu=33.56ms elapsed=38.81s tid=0x00007fbe280a3800 nid=0x1b05 runnable "G1 Conc#1" os_prio=0 cpu=35.79ms elapsed=38.13s tid=0x00007fbdf877a800 nid=0x1b1b runnable "G1 Refine#0" os_prio=0 cpu=6.18ms elapsed=38.81s tid=0x00007fbe2822a000 nid=0x1b06 runnable "G1 Young RemSet Sampling" os_prio=0 cpu=17.10ms elapsed=38.81s tid=0x00007fbe2822c000 nid=0x1b07 runnable "VM Periodic Task Thread" os_prio=0 cpu=29.41ms elapsed=38.75s tid=0x00007fbe283ba000 nid=0x1b11 waiting on condition JNI global refs: 18, weak refs: 0