2022-03-11 20:51:48 Full thread dump OpenJDK 64-Bit Server VM (25.265-b01 mixed mode): "IPC Client (1089680530) connection to cvp365.sjc.aristanetworks.com/172.30.41.155:9001 from cvp" #23455 daemon prio=5 os_prio=0 tid=0x00005650685b8000 nid=0x4390 in Object.wait() [0x00007fccdb0f5000] java.lang.Thread.State: TIMED_WAITING (on object monitor) at java.lang.Object.wait(Native Method) at org.apache.hadoop.ipc.Client$Connection.waitForWork(Client.java:1042) - locked <0x00000000c404c238> (a org.apache.hadoop.ipc.Client$Connection) at org.apache.hadoop.ipc.Client$Connection.run(Client.java:1086) Locked ownable synchronizers: - None "Attach Listener" #23451 daemon prio=9 os_prio=0 tid=0x00007fcd26ea0000 nid=0x5dbd waiting on condition [0x0000000000000000] java.lang.Thread.State: RUNNABLE Locked ownable synchronizers: - None "Timer for 'HBase' metrics system" #23446 daemon prio=5 os_prio=0 tid=0x0000565068733000 nid=0x8f0 in Object.wait() [0x00007fccdacf1000] java.lang.Thread.State: TIMED_WAITING (on object monitor) at java.lang.Object.wait(Native Method) at java.util.TimerThread.mainLoop(Timer.java:552) - locked <0x00000000b7845e48> (a java.util.TaskQueue) at java.util.TimerThread.run(Timer.java:505) Locked ownable synchronizers: - None "IPC Parameter Sending Thread #5" #21038 daemon prio=5 os_prio=0 tid=0x0000565068bc9000 nid=0x1a4 waiting on condition [0x00007fcce03f3000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00000000a2400098> (a java.util.concurrent.SynchronousQueue$TransferStack) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460) at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362) at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "Connector-Scheduler-2fc2a205-1" #19931 prio=5 os_prio=0 tid=0x0000565068356800 nid=0x4325 waiting on condition [0x00007fcce02f2000] java.lang.Thread.State: WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00000000a2a8cbb8> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1081) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "RS_LOG_REPLAY_OPS-regionserver/cvp365:16201-1" #14349 daemon prio=5 os_prio=0 tid=0x00007fcd26f6a800 nid=0x2a8f waiting on condition [0x00007fccdb9fc000] java.lang.Thread.State: WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00000000a2c0d078> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "RS_LOG_REPLAY_OPS-regionserver/cvp365:16201-0" #14348 daemon prio=5 os_prio=0 tid=0x00007fcd26a78000 nid=0x2a8e waiting on condition [0x00007fcce01f1000] java.lang.Thread.State: WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00000000a2c0d078> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "regionserver/cvp365:16201-shortCompactions-0" #594 daemon prio=5 os_prio=0 tid=0x00007fcd26935800 nid=0x4286 waiting on condition [0x00007fccf4556000] java.lang.Thread.State: WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00000000a2a80200> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) at java.util.concurrent.PriorityBlockingQueue.take(PriorityBlockingQueue.java:549) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "Log-Archiver-0" #439 daemon prio=5 os_prio=0 tid=0x00007fcd11059000 nid=0x8bb waiting on condition [0x00007fcce04f4000] java.lang.Thread.State: WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00000000a0083498> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "regionserver/cvp365:16201.Chore.3" #388 daemon prio=5 os_prio=0 tid=0x00007fcd103b6000 nid=0x574 waiting on condition [0x00007fcce6f2f000] java.lang.Thread.State: WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00000000a236cfc8> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1088) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "regionserver/cvp365:16201.Chore.2" #386 daemon prio=5 os_prio=0 tid=0x00007fcd08a15800 nid=0x539 waiting on condition [0x00007fcce05f5000] java.lang.Thread.State: WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00000000a236cfc8> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1088) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "pool-1-thread-5" #265 daemon prio=5 os_prio=0 tid=0x00007fcd11001000 nid=0x1ba waiting on condition [0x00007fcce08f8000] java.lang.Thread.State: WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00000000a1a6b6a0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "pool-1-thread-4" #262 daemon prio=5 os_prio=0 tid=0x00007fcd111fa000 nid=0x4cee waiting on condition [0x00007fcce07f7000] java.lang.Thread.State: WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00000000a1a6b6a0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "pool-1-thread-3" #257 daemon prio=5 os_prio=0 tid=0x00007fcd10a63000 nid=0x1bf1 waiting on condition [0x00007fcce09f9000] java.lang.Thread.State: WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00000000a1a6b6a0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "pool-1-thread-2" #254 daemon prio=5 os_prio=0 tid=0x00007fcd10b5e000 nid=0x554a waiting on condition [0x00007fcce0afa000] java.lang.Thread.State: WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00000000a1a6b6a0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "pool-1-thread-1" #249 daemon prio=5 os_prio=0 tid=0x00007fcd111fc800 nid=0x6ccb waiting on condition [0x00007fcce0dfb000] java.lang.Thread.State: WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00000000a1a6b6a0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "ShortCircuitCache_SlotReleaser" #246 daemon prio=5 os_prio=0 tid=0x00005650684fb000 nid=0x2e36 waiting on condition [0x00007fcce1921000] java.lang.Thread.State: WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00000000a2445680> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1081) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "RS_COMPACTED_FILES_DISCHARGER-regionserver/cvp365:16201-9" #239 daemon prio=5 os_prio=0 tid=0x00007fcd081d6800 nid=0x252f waiting on condition [0x00007fcce1a22000] java.lang.Thread.State: WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00000000a28e42b8> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "RS_COMPACTED_FILES_DISCHARGER-regionserver/cvp365:16201-8" #238 daemon prio=5 os_prio=0 tid=0x00007fcd081d5800 nid=0x252e waiting on condition [0x00007fcce1d23000] java.lang.Thread.State: WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00000000a28e42b8> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "RS_COMPACTED_FILES_DISCHARGER-regionserver/cvp365:16201-7" #237 daemon prio=5 os_prio=0 tid=0x00007fcd08266000 nid=0x252d waiting on condition [0x00007fcce1e24000] java.lang.Thread.State: WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00000000a28e42b8> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "RS_COMPACTED_FILES_DISCHARGER-regionserver/cvp365:16201-6" #236 daemon prio=5 os_prio=0 tid=0x00007fcd08198000 nid=0x252c waiting on condition [0x00007fcce2e1a000] java.lang.Thread.State: WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00000000a28e42b8> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "RS_COMPACTED_FILES_DISCHARGER-regionserver/cvp365:16201-5" #235 daemon prio=5 os_prio=0 tid=0x00007fcd0833c800 nid=0x252b waiting on condition [0x00007fcce1f25000] java.lang.Thread.State: WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00000000a28e42b8> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "RS_COMPACTED_FILES_DISCHARGER-regionserver/cvp365:16201-4" #234 daemon prio=5 os_prio=0 tid=0x00007fcd081d4800 nid=0x252a waiting on condition [0x00007fcce2f1b000] java.lang.Thread.State: WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00000000a28e42b8> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "RS_COMPACTED_FILES_DISCHARGER-regionserver/cvp365:16201-3" #233 daemon prio=5 os_prio=0 tid=0x00007fcd081ba000 nid=0x2529 waiting on condition [0x00007fcce301c000] java.lang.Thread.State: WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00000000a28e42b8> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "RS_COMPACTED_FILES_DISCHARGER-regionserver/cvp365:16201-2" #232 daemon prio=5 os_prio=0 tid=0x00007fcd084a7000 nid=0x2528 waiting on condition [0x00007fcce3c29000] java.lang.Thread.State: WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00000000a28e42b8> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "RS_CLOSE_REGION-regionserver/cvp365:16201-2" #150 daemon prio=5 os_prio=0 tid=0x0000565068450800 nid=0x7aad waiting on condition [0x00007fcce351f000] java.lang.Thread.State: WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00000000a28e4370> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "RS_CLOSE_REGION-regionserver/cvp365:16201-1" #149 daemon prio=5 os_prio=0 tid=0x000056506844f800 nid=0x7aac waiting on condition [0x00007fcce331d000] java.lang.Thread.State: WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00000000a28e4370> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "RS_CLOSE_REGION-regionserver/cvp365:16201-0" #148 daemon prio=5 os_prio=0 tid=0x0000565068505800 nid=0x7aab waiting on condition [0x00007fcce341e000] java.lang.Thread.State: WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00000000a28e4370> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "RS_OPEN_REGION-regionserver/cvp365:16201-2" #132 daemon prio=5 os_prio=0 tid=0x00007fcd25e9a800 nid=0x75cb runnable [0x00007fcce3925000] java.lang.Thread.State: RUNNABLE at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:338) at com.lmax.disruptor.MultiProducerSequencer.next(MultiProducerSequencer.java:136) at com.lmax.disruptor.MultiProducerSequencer.next(MultiProducerSequencer.java:105) at com.lmax.disruptor.RingBuffer.next(RingBuffer.java:263) at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$stampSequenceIdAndPublishToRingBuffer$2(AbstractFSWAL.java:1086) at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL$$Lambda$179/1460879368.run(Unknown Source) at org.apache.hadoop.hbase.regionserver.MultiVersionConcurrencyControl.begin(MultiVersionConcurrencyControl.java:145) - locked <0x00000000a4b24580> (a java.util.LinkedList) at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.stampSequenceIdAndPublishToRingBuffer(AbstractFSWAL.java:1085) at org.apache.hadoop.hbase.regionserver.wal.AsyncFSWAL.append(AsyncFSWAL.java:607) at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendMarker(AbstractFSWAL.java:1145) at org.apache.hadoop.hbase.regionserver.wal.WALUtil.doFullMarkerAppendTransaction(WALUtil.java:161) at org.apache.hadoop.hbase.regionserver.wal.WALUtil.writeMarker(WALUtil.java:140) at org.apache.hadoop.hbase.regionserver.wal.WALUtil.writeRegionEventMarker(WALUtil.java:105) at org.apache.hadoop.hbase.regionserver.HRegion.writeRegionOpenMarker(HRegion.java:1251) at org.apache.hadoop.hbase.regionserver.HRegion.openHRegion(HRegion.java:8138) at org.apache.hadoop.hbase.regionserver.HRegion.openHRegionFromTableDir(HRegion.java:8090) at org.apache.hadoop.hbase.regionserver.HRegion.openHRegion(HRegion.java:8066) at org.apache.hadoop.hbase.regionserver.HRegion.openHRegion(HRegion.java:8024) at org.apache.hadoop.hbase.regionserver.HRegion.openHRegion(HRegion.java:7975) at org.apache.hadoop.hbase.regionserver.handler.AssignRegionHandler.process(AssignRegionHandler.java:145) at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - <0x00000000a2c97168> (a java.util.concurrent.ThreadPoolExecutor$Worker) "RS_OPEN_REGION-regionserver/cvp365:16201-1" #131 daemon prio=5 os_prio=0 tid=0x00007fcd25b83800 nid=0x75ca runnable [0x00007fcce3a26000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:338) at com.lmax.disruptor.MultiProducerSequencer.next(MultiProducerSequencer.java:136) at com.lmax.disruptor.MultiProducerSequencer.next(MultiProducerSequencer.java:105) at com.lmax.disruptor.RingBuffer.next(RingBuffer.java:263) at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$stampSequenceIdAndPublishToRingBuffer$2(AbstractFSWAL.java:1086) at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL$$Lambda$179/1460879368.run(Unknown Source) at org.apache.hadoop.hbase.regionserver.MultiVersionConcurrencyControl.begin(MultiVersionConcurrencyControl.java:145) - locked <0x00000000a4b8d100> (a java.util.LinkedList) at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.stampSequenceIdAndPublishToRingBuffer(AbstractFSWAL.java:1085) at org.apache.hadoop.hbase.regionserver.wal.AsyncFSWAL.append(AsyncFSWAL.java:607) at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendMarker(AbstractFSWAL.java:1145) at org.apache.hadoop.hbase.regionserver.wal.WALUtil.doFullMarkerAppendTransaction(WALUtil.java:161) at org.apache.hadoop.hbase.regionserver.wal.WALUtil.writeMarker(WALUtil.java:140) at org.apache.hadoop.hbase.regionserver.wal.WALUtil.writeRegionEventMarker(WALUtil.java:105) at org.apache.hadoop.hbase.regionserver.HRegion.writeRegionOpenMarker(HRegion.java:1251) at org.apache.hadoop.hbase.regionserver.HRegion.openHRegion(HRegion.java:8138) at org.apache.hadoop.hbase.regionserver.HRegion.openHRegionFromTableDir(HRegion.java:8090) at org.apache.hadoop.hbase.regionserver.HRegion.openHRegion(HRegion.java:8066) at org.apache.hadoop.hbase.regionserver.HRegion.openHRegion(HRegion.java:8024) at org.apache.hadoop.hbase.regionserver.HRegion.openHRegion(HRegion.java:7975) at org.apache.hadoop.hbase.regionserver.handler.AssignRegionHandler.process(AssignRegionHandler.java:145) at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - <0x00000000a2c97310> (a java.util.concurrent.ThreadPoolExecutor$Worker) "RS_COMPACTED_FILES_DISCHARGER-regionserver/cvp365:16201-1" #127 daemon prio=5 os_prio=0 tid=0x00007fcd080bf800 nid=0x421f waiting on condition [0x00007fccf5d5d000] java.lang.Thread.State: WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00000000a28e42b8> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "RS_COMPACTED_FILES_DISCHARGER-regionserver/cvp365:16201-0" #126 daemon prio=5 os_prio=0 tid=0x00007fcd080be800 nid=0x4208 waiting on condition [0x00007fccf5c5c000] java.lang.Thread.State: WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00000000a28e42b8> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "RS-EventLoopGroup-1-16" #123 daemon prio=10 os_prio=0 tid=0x00007fcd081c6800 nid=0x9f0 runnable [0x00007fcce3b28000] java.lang.Thread.State: RUNNABLE at org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) at org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:192) at org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:185) at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:290) at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:347) at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:989) at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "RS-EventLoopGroup-1-15" #121 daemon prio=10 os_prio=0 tid=0x00007fcd081c5000 nid=0x57b runnable [0x00007fcce3d2a000] java.lang.Thread.State: RUNNABLE at org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) at org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:192) at org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:185) at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:290) at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:347) at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:989) at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "RS-EventLoopGroup-1-14" #120 daemon prio=10 os_prio=0 tid=0x00007fcd0834b800 nid=0x7eed runnable [0x00007fccf4657000] java.lang.Thread.State: RUNNABLE at org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) at org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:192) at org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:185) at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:290) at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:347) at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:989) at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "RS-EventLoopGroup-1-13" #119 daemon prio=10 os_prio=0 tid=0x00007fcd081c3000 nid=0x7866 runnable [0x00007fcce6e2e000] java.lang.Thread.State: RUNNABLE at org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) at org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:192) at org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:185) at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:290) at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:347) at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:989) at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "RS-EventLoopGroup-1-12" #117 daemon prio=10 os_prio=0 tid=0x00007fcd081c1800 nid=0x7447 runnable [0x00007fccfc127000] java.lang.Thread.State: RUNNABLE at org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) at org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:192) at org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:185) at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:290) at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:347) at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:989) at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "RS-EventLoopGroup-1-11" #116 daemon prio=10 os_prio=0 tid=0x00007fcd081c0000 nid=0x7384 runnable [0x00007fcce9032000] java.lang.Thread.State: RUNNABLE at org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) at org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:192) at org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:185) at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:290) at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:347) at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:989) at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "RS-EventLoopGroup-1-10" #115 daemon prio=10 os_prio=0 tid=0x00007fcd08232000 nid=0x711f runnable [0x00007fccea134000] java.lang.Thread.State: RUNNABLE at org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) at org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:192) at org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:185) at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:290) at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:347) at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:989) at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "RS_OPEN_REGION-regionserver/cvp365:16201-0" #112 daemon prio=5 os_prio=0 tid=0x00007fcd1031a000 nid=0x711c runnable [0x00007fccea434000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:338) at com.lmax.disruptor.MultiProducerSequencer.next(MultiProducerSequencer.java:136) at com.lmax.disruptor.MultiProducerSequencer.next(MultiProducerSequencer.java:105) at com.lmax.disruptor.RingBuffer.next(RingBuffer.java:263) at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$stampSequenceIdAndPublishToRingBuffer$2(AbstractFSWAL.java:1086) at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL$$Lambda$179/1460879368.run(Unknown Source) at org.apache.hadoop.hbase.regionserver.MultiVersionConcurrencyControl.begin(MultiVersionConcurrencyControl.java:145) - locked <0x00000000a4b6a198> (a java.util.LinkedList) at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.stampSequenceIdAndPublishToRingBuffer(AbstractFSWAL.java:1085) at org.apache.hadoop.hbase.regionserver.wal.AsyncFSWAL.append(AsyncFSWAL.java:607) at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendMarker(AbstractFSWAL.java:1145) at org.apache.hadoop.hbase.regionserver.wal.WALUtil.doFullMarkerAppendTransaction(WALUtil.java:161) at org.apache.hadoop.hbase.regionserver.wal.WALUtil.writeMarker(WALUtil.java:140) at org.apache.hadoop.hbase.regionserver.wal.WALUtil.writeRegionEventMarker(WALUtil.java:105) at org.apache.hadoop.hbase.regionserver.HRegion.writeRegionOpenMarker(HRegion.java:1251) at org.apache.hadoop.hbase.regionserver.HRegion.openHRegion(HRegion.java:8138) at org.apache.hadoop.hbase.regionserver.HRegion.openHRegionFromTableDir(HRegion.java:8090) at org.apache.hadoop.hbase.regionserver.HRegion.openHRegion(HRegion.java:8066) at org.apache.hadoop.hbase.regionserver.HRegion.openHRegion(HRegion.java:8024) at org.apache.hadoop.hbase.regionserver.HRegion.openHRegion(HRegion.java:7975) at org.apache.hadoop.hbase.regionserver.handler.AssignRegionHandler.process(AssignRegionHandler.java:145) at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - <0x00000000a2c984c8> (a java.util.concurrent.ThreadPoolExecutor$Worker) "RS-EventLoopGroup-1-9" #111 daemon prio=10 os_prio=0 tid=0x00007fcd0832e000 nid=0x70fc runnable [0x00007fccec738000] java.lang.Thread.State: RUNNABLE at org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) at org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:192) at org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:185) at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:290) at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:347) at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:989) at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "RS_OPEN_PRIORITY_REGION-regionserver/cvp365:16201-0" #108 daemon prio=5 os_prio=0 tid=0x000056506839d000 nid=0x70f9 waiting on condition [0x00007fcceca39000] java.lang.Thread.State: WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00000000a28e44e0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "ShortCircuitCache_Cleaner" #107 daemon prio=5 os_prio=0 tid=0x000056506846e000 nid=0x70f8 waiting on condition [0x00007fccecb3a000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00000000a2400000> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1093) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "org.apache.hadoop.hdfs.PeerCache@1e8bb991" #106 daemon prio=5 os_prio=0 tid=0x000056506846b800 nid=0x70f7 waiting on condition [0x00007fccecc3b000] java.lang.Thread.State: TIMED_WAITING (sleeping) at java.lang.Thread.sleep(Native Method) at org.apache.hadoop.hdfs.PeerCache.run(PeerCache.java:253) at org.apache.hadoop.hdfs.PeerCache.access$000(PeerCache.java:46) at org.apache.hadoop.hdfs.PeerCache$1.run(PeerCache.java:124) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "Monitor thread for TaskMonitor" #105 daemon prio=5 os_prio=0 tid=0x0000565068475000 nid=0x70f4 waiting on condition [0x00007fccecd3c000] java.lang.Thread.State: TIMED_WAITING (sleeping) at java.lang.Thread.sleep(Native Method) at org.apache.hadoop.hbase.monitoring.TaskMonitor$MonitorRunnable.run(TaskMonitor.java:302) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "RS-EventLoopGroup-1-8" #104 daemon prio=10 os_prio=0 tid=0x00007fcd08009800 nid=0x70f3 runnable [0x00007fccede3e000] java.lang.Thread.State: RUNNABLE at org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) at org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:192) at org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:185) at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:290) at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:347) at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:989) at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "AsyncFSWAL-0-hdfs://mycluster/hbase" #103 daemon prio=5 os_prio=0 tid=0x00007fcd103b0000 nid=0x70e2 waiting on condition [0x00007fccedf3f000] java.lang.Thread.State: WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00000000a28e9598> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "RS-EventLoopGroup-1-7" #102 daemon prio=10 os_prio=0 tid=0x00007fcd1002f800 nid=0x70de runnable [0x00007fccf2244000] java.lang.Thread.State: RUNNABLE at org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) at org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:192) at org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:185) at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:290) at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:347) at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:989) at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "RS-EventLoopGroup-1-6" #101 daemon prio=10 os_prio=0 tid=0x00007fcd1056c800 nid=0x70dc runnable [0x00007fccf2345000] java.lang.Thread.State: RUNNABLE at org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) at org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:192) at org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:185) at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:290) at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:347) at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:989) at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "RS-EventLoopGroup-1-5" #100 daemon prio=10 os_prio=0 tid=0x00007fcd10569800 nid=0x70db runnable [0x00007fccf2446000] java.lang.Thread.State: RUNNABLE at org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) at org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:192) at org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:185) at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:290) at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:347) at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:989) at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "LeaseRenewer:cvp@mycluster" #99 daemon prio=5 os_prio=0 tid=0x00007fcd102c3000 nid=0x70da waiting on condition [0x00007fccf2547000] java.lang.Thread.State: TIMED_WAITING (sleeping) at java.lang.Thread.sleep(Native Method) at org.apache.hadoop.hdfs.client.impl.LeaseRenewer.run(LeaseRenewer.java:412) at org.apache.hadoop.hdfs.client.impl.LeaseRenewer.access$600(LeaseRenewer.java:76) at org.apache.hadoop.hdfs.client.impl.LeaseRenewer$1.run(LeaseRenewer.java:308) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "RS-EventLoopGroup-1-4" #98 daemon prio=10 os_prio=0 tid=0x00007fcd0800d800 nid=0x6f0d runnable [0x00007fccf3649000] java.lang.Thread.State: RUNNABLE at org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) at org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:192) at org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:185) at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:290) at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:347) at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:989) at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "cvp365:16201Replication Statistics #0" #97 daemon prio=5 os_prio=0 tid=0x00007fcd107f8800 nid=0x6c71 waiting on condition [0x00007fccf374a000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00000000a28ea028> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1093) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "ReplicationExecutor-0" #96 daemon prio=5 os_prio=0 tid=0x00007fcd107f6800 nid=0x6c70 waiting on condition [0x00007fccf384b000] java.lang.Thread.State: WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00000000a004a108> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "regionserver/cvp365:16201-MemStoreChunkPool Statistics" #95 daemon prio=5 os_prio=0 tid=0x00007fcd107f4000 nid=0x6c6f waiting on condition [0x00007fccf394c000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00000000a28ea288> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1093) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "regionserver/cvp365:16201-MemStoreChunkPool Statistics" #93 daemon prio=5 os_prio=0 tid=0x00007fcd107f3000 nid=0x6c6e waiting on condition [0x00007fccf3a4d000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00000000a28ea4e8> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1093) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "regionserver/cvp365:16201.leaseChecker" #85 daemon prio=5 os_prio=0 tid=0x00007fcd107dd000 nid=0x6c6d waiting on condition [0x00007fccf3b4e000] java.lang.Thread.State: TIMED_WAITING (sleeping) at java.lang.Thread.sleep(Native Method) at org.apache.hadoop.hbase.regionserver.LeaseManager.run(LeaseManager.java:90) Locked ownable synchronizers: - None "regionserver/cvp365:16201.procedureResultReporter" #87 daemon prio=5 os_prio=0 tid=0x00007fcd107db000 nid=0x6c6c waiting on condition [0x00007fccf3c4f000] java.lang.Thread.State: WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00000000a28ea5d0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) at org.apache.hadoop.hbase.regionserver.RemoteProcedureResultReporter.run(RemoteProcedureResultReporter.java:77) Locked ownable synchronizers: - None "MemStoreFlusher.1" #90 prio=5 os_prio=0 tid=0x00007fcd107d8800 nid=0x6c6b waiting on condition [0x00007fccf3d50000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00000000a28ea998> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.DelayQueue.poll(DelayQueue.java:259) at java.util.concurrent.DelayQueue.poll(DelayQueue.java:70) at org.apache.hadoop.hbase.regionserver.MemStoreFlusher$FlushHandler.run(MemStoreFlusher.java:334) Locked ownable synchronizers: - None "MemStoreFlusher.0" #88 prio=5 os_prio=0 tid=0x00007fcd107d5000 nid=0x6c6a waiting on condition [0x00007fccf3e51000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00000000a28ea998> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.DelayQueue.poll(DelayQueue.java:259) at java.util.concurrent.DelayQueue.poll(DelayQueue.java:70) at org.apache.hadoop.hbase.regionserver.MemStoreFlusher$FlushHandler.run(MemStoreFlusher.java:334) Locked ownable synchronizers: - None "regionserver/cvp365:16201.logRoller" #86 daemon prio=5 os_prio=0 tid=0x00007fcd107d6800 nid=0x6c69 waiting on condition [0x00007fccf3f52000] java.lang.Thread.State: WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00000000aca70fb8> (a java.util.concurrent.CompletableFuture$Signaller) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.CompletableFuture$Signaller.block(CompletableFuture.java:1707) at java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3323) at java.util.concurrent.CompletableFuture.waitingGet(CompletableFuture.java:1742) at java.util.concurrent.CompletableFuture.get(CompletableFuture.java:1908) at org.apache.hadoop.hbase.regionserver.wal.AsyncProtobufLogWriter.write(AsyncProtobufLogWriter.java:189) at org.apache.hadoop.hbase.regionserver.wal.AsyncProtobufLogWriter.writeMagicAndWALHeader(AsyncProtobufLogWriter.java:202) at org.apache.hadoop.hbase.regionserver.wal.AbstractProtobufLogWriter.init(AbstractProtobufLogWriter.java:170) at org.apache.hadoop.hbase.wal.AsyncFSWALProvider.createAsyncWriter(AsyncFSWALProvider.java:113) at org.apache.hadoop.hbase.regionserver.wal.AsyncFSWAL.createWriterInstance(AsyncFSWAL.java:669) at org.apache.hadoop.hbase.regionserver.wal.AsyncFSWAL.createWriterInstance(AsyncFSWAL.java:130) at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.rollWriter(AbstractFSWAL.java:841) at org.apache.hadoop.hbase.wal.AbstractWALRoller$RollController.rollWal(AbstractWALRoller.java:268) at org.apache.hadoop.hbase.wal.AbstractWALRoller.run(AbstractWALRoller.java:187) Locked ownable synchronizers: - <0x00000000a007f198> (a java.util.concurrent.locks.ReentrantLock$FairSync) "regionserver/cvp365:16201.Chore.1" #84 daemon prio=5 os_prio=0 tid=0x00007fcd107a6000 nid=0x6c68 waiting on condition [0x00007fccf4253000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00000000a236cfc8> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1093) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "regionserver/cvp365:16201-longCompactions-0" #83 daemon prio=5 os_prio=0 tid=0x00007fcd107a1000 nid=0x6c67 waiting on condition [0x00007fccf4354000] java.lang.Thread.State: WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00000000a28eacf0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) at org.apache.hadoop.hbase.util.StealJobQueue.take(StealJobQueue.java:106) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "JvmPauseMonitor" #82 daemon prio=5 os_prio=0 tid=0x00007fcd1079a800 nid=0x6c66 waiting on condition [0x00007fccf4455000] java.lang.Thread.State: TIMED_WAITING (sleeping) at java.lang.Thread.sleep(Native Method) at org.apache.hadoop.hbase.util.JvmPauseMonitor$Monitor.run(JvmPauseMonitor.java:154) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "RpcClient-timer-pool-0" #76 daemon prio=5 os_prio=0 tid=0x00007fcd26ff1800 nid=0x6c58 sleeping[0x00007fccf5959000] java.lang.Thread.State: TIMED_WAITING (sleeping) at java.lang.Thread.sleep(Native Method) at org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$Worker.waitForNextTick(HashedWheelTimer.java:566) at org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$Worker.run(HashedWheelTimer.java:462) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "RS-EventLoopGroup-1-3" #79 daemon prio=10 os_prio=0 tid=0x00007fcd106ae800 nid=0x6c57 runnable [0x00007fccf5a5a000] java.lang.Thread.State: RUNNABLE at org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) at org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:192) at org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:185) at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:290) at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:347) at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:989) at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "Idle-Rpc-Conn-Sweeper-pool-0" #77 daemon prio=5 os_prio=0 tid=0x00007fcd10620800 nid=0x6c2d runnable [0x00007fccf5b5b000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00000000a28eb128> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1093) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "ReadOnlyZKClient-cvp328.sjc.aristanetworks.com:2181,cvp365.sjc.aristanetworks.com:2181,cvp90.sjc.aristanetworks.com:2181@0x1708f354" #73 daemon prio=5 os_prio=0 tid=0x00007fcd105fc800 nid=0x6c2a waiting on condition [0x00007fccf605e000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00000000a28eb518> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.DelayQueue.poll(DelayQueue.java:259) at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:323) at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$$Lambda$63/381340713.run(Unknown Source) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "regionserver/cvp365:16201" #17 prio=5 os_prio=0 tid=0x00007fcd26fad000 nid=0x6bed in Object.wait() [0x00007fccf615f000] java.lang.Thread.State: TIMED_WAITING (on object monitor) at java.lang.Object.wait(Native Method) at org.apache.hadoop.hbase.util.Sleeper.sleep(Sleeper.java:84) - locked <0x00000000a01578c0> (a java.lang.Object) at org.apache.hadoop.hbase.util.Sleeper.sleep(Sleeper.java:67) at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:1096) Locked ownable synchronizers: - None "Session-HouseKeeper-1976f537-1" #72 prio=5 os_prio=0 tid=0x00007fcd26def800 nid=0x6be7 waiting on condition [0x00007fccf6260000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00000000a28eb718> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1093) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "qtp957387062-71" #71 daemon prio=5 os_prio=0 tid=0x00007fcd2665e000 nid=0x6bc6 waiting on condition [0x00007fccf6561000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00000000a24000f8> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at org.apache.hbase.thirdparty.org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:382) at org.apache.hbase.thirdparty.org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.idleJobPoll(QueuedThreadPool.java:973) at org.apache.hbase.thirdparty.org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:1023) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "qtp957387062-70" #70 daemon prio=5 os_prio=0 tid=0x00007fcd2665c000 nid=0x6bc5 waiting on condition [0x00007fccf6662000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00000000a24000f8> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at org.apache.hbase.thirdparty.org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:382) at org.apache.hbase.thirdparty.org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.idleJobPoll(QueuedThreadPool.java:973) at org.apache.hbase.thirdparty.org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:1023) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "qtp957387062-69" #69 daemon prio=5 os_prio=0 tid=0x00007fcd2665a800 nid=0x6bc4 runnable [0x00007fccf6763000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00000000a28ec4c8> (a sun.nio.ch.Util$3) - locked <0x00000000a28ec4d8> (a java.util.Collections$UnmodifiableSet) - locked <0x00000000a28ec480> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:101) at org.apache.hbase.thirdparty.org.eclipse.jetty.io.ManagedSelector.nioSelect(ManagedSelector.java:183) at org.apache.hbase.thirdparty.org.eclipse.jetty.io.ManagedSelector.select(ManagedSelector.java:190) at org.apache.hbase.thirdparty.org.eclipse.jetty.io.ManagedSelector$SelectorProducer.select(ManagedSelector.java:606) at org.apache.hbase.thirdparty.org.eclipse.jetty.io.ManagedSelector$SelectorProducer.produce(ManagedSelector.java:543) at org.apache.hbase.thirdparty.org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produceTask(EatWhatYouKill.java:360) at org.apache.hbase.thirdparty.org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.doProduce(EatWhatYouKill.java:184) at org.apache.hbase.thirdparty.org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.tryProduce(EatWhatYouKill.java:171) at org.apache.hbase.thirdparty.org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.run(EatWhatYouKill.java:129) at org.apache.hbase.thirdparty.org.eclipse.jetty.util.thread.ReservedThreadExecutor$ReservedThread.run(ReservedThreadExecutor.java:383) at org.apache.hbase.thirdparty.org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:882) at org.apache.hbase.thirdparty.org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:1036) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "qtp957387062-68" #68 daemon prio=5 os_prio=0 tid=0x00007fcd26658800 nid=0x6bc3 waiting on condition [0x00007fccf6864000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00000000a24000f8> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at org.apache.hbase.thirdparty.org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:382) at org.apache.hbase.thirdparty.org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.idleJobPoll(QueuedThreadPool.java:973) at org.apache.hbase.thirdparty.org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:1023) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "qtp957387062-67" #67 daemon prio=5 os_prio=0 tid=0x00007fcd245c6800 nid=0x6bc2 waiting on condition [0x00007fccf6965000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00000000a24000f8> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at org.apache.hbase.thirdparty.org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:382) at org.apache.hbase.thirdparty.org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.idleJobPoll(QueuedThreadPool.java:973) at org.apache.hbase.thirdparty.org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:1023) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "qtp957387062-66" #66 daemon prio=5 os_prio=0 tid=0x00007fcd245c4800 nid=0x6bc1 waiting on condition [0x00007fccf6a66000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00000000a24000f8> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at org.apache.hbase.thirdparty.org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:382) at org.apache.hbase.thirdparty.org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.idleJobPoll(QueuedThreadPool.java:973) at org.apache.hbase.thirdparty.org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:1023) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "qtp957387062-65-acceptor-0@26edc1c7-ServerConnector@2fc2a205{HTTP/1.1, (http/1.1)}{0.0.0.0:16301}" #65 daemon prio=3 os_prio=0 tid=0x00007fcd245c2800 nid=0x6bc0 runnable [0x00007fccf6b67000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.ServerSocketChannelImpl.accept0(Native Method) at sun.nio.ch.ServerSocketChannelImpl.accept(ServerSocketChannelImpl.java:419) at sun.nio.ch.ServerSocketChannelImpl.accept(ServerSocketChannelImpl.java:247) - locked <0x00000000a28ec1c8> (a java.lang.Object) at org.apache.hbase.thirdparty.org.eclipse.jetty.server.ServerConnector.accept(ServerConnector.java:388) at org.apache.hbase.thirdparty.org.eclipse.jetty.server.AbstractConnector$Acceptor.run(AbstractConnector.java:702) at org.apache.hbase.thirdparty.org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:882) at org.apache.hbase.thirdparty.org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:1036) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "qtp957387062-64" #64 daemon prio=5 os_prio=0 tid=0x00007fcd245c0800 nid=0x6bbf waiting on condition [0x00007fccf6c68000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00000000a24000f8> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at org.apache.hbase.thirdparty.org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:382) at org.apache.hbase.thirdparty.org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.idleJobPoll(QueuedThreadPool.java:973) at org.apache.hbase.thirdparty.org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:1023) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "RpcServer.metaPriority.FPBQ.Fifo.handler=0,queue=0,port=16201" #63 daemon prio=5 os_prio=0 tid=0x00007fcd264aa800 nid=0x6b97 waiting on condition [0x00007fccf6f69000] java.lang.Thread.State: WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00000000a2400368> (a java.util.concurrent.Semaphore$NonfairSync) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer.parkAndCheckInterrupt(AbstractQueuedSynchronizer.java:836) at java.util.concurrent.locks.AbstractQueuedSynchronizer.doAcquireSharedInterruptibly(AbstractQueuedSynchronizer.java:997) at java.util.concurrent.locks.AbstractQueuedSynchronizer.acquireSharedInterruptibly(AbstractQueuedSynchronizer.java:1304) at java.util.concurrent.Semaphore.acquire(Semaphore.java:312) at org.apache.hadoop.hbase.ipc.FastPathBalancedQueueRpcExecutor$FastPathHandler.getCallRunner(FastPathBalancedQueueRpcExecutor.java:109) at org.apache.hadoop.hbase.ipc.RpcExecutor$Handler.run(RpcExecutor.java:334) Locked ownable synchronizers: - None "RpcServer.replication.FPBQ.Fifo.handler=2,queue=0,port=16201" #62 daemon prio=5 os_prio=0 tid=0x00007fcd264a8800 nid=0x6b96 waiting on condition [0x00007fccf706a000] java.lang.Thread.State: WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00000000a2400578> (a java.util.concurrent.Semaphore$NonfairSync) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer.parkAndCheckInterrupt(AbstractQueuedSynchronizer.java:836) at java.util.concurrent.locks.AbstractQueuedSynchronizer.doAcquireSharedInterruptibly(AbstractQueuedSynchronizer.java:997) at java.util.concurrent.locks.AbstractQueuedSynchronizer.acquireSharedInterruptibly(AbstractQueuedSynchronizer.java:1304) at java.util.concurrent.Semaphore.acquire(Semaphore.java:312) at org.apache.hadoop.hbase.ipc.FastPathBalancedQueueRpcExecutor$FastPathHandler.getCallRunner(FastPathBalancedQueueRpcExecutor.java:109) at org.apache.hadoop.hbase.ipc.RpcExecutor$Handler.run(RpcExecutor.java:334) Locked ownable synchronizers: - None "RpcServer.replication.FPBQ.Fifo.handler=1,queue=0,port=16201" #61 daemon prio=5 os_prio=0 tid=0x00007fcd264a6800 nid=0x6b95 waiting on condition [0x00007fccf716b000] java.lang.Thread.State: WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00000000a2400788> (a java.util.concurrent.Semaphore$NonfairSync) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer.parkAndCheckInterrupt(AbstractQueuedSynchronizer.java:836) at java.util.concurrent.locks.AbstractQueuedSynchronizer.doAcquireSharedInterruptibly(AbstractQueuedSynchronizer.java:997) at java.util.concurrent.locks.AbstractQueuedSynchronizer.acquireSharedInterruptibly(AbstractQueuedSynchronizer.java:1304) at java.util.concurrent.Semaphore.acquire(Semaphore.java:312) at org.apache.hadoop.hbase.ipc.FastPathBalancedQueueRpcExecutor$FastPathHandler.getCallRunner(FastPathBalancedQueueRpcExecutor.java:109) at org.apache.hadoop.hbase.ipc.RpcExecutor$Handler.run(RpcExecutor.java:334) Locked ownable synchronizers: - None "RpcServer.replication.FPBQ.Fifo.handler=0,queue=0,port=16201" #60 daemon prio=5 os_prio=0 tid=0x00007fcd264a4800 nid=0x6b94 waiting on condition [0x00007fccf726c000] java.lang.Thread.State: WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00000000a2400998> (a java.util.concurrent.Semaphore$NonfairSync) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer.parkAndCheckInterrupt(AbstractQueuedSynchronizer.java:836) at java.util.concurrent.locks.AbstractQueuedSynchronizer.doAcquireSharedInterruptibly(AbstractQueuedSynchronizer.java:997) at java.util.concurrent.locks.AbstractQueuedSynchronizer.acquireSharedInterruptibly(AbstractQueuedSynchronizer.java:1304) at java.util.concurrent.Semaphore.acquire(Semaphore.java:312) at org.apache.hadoop.hbase.ipc.FastPathBalancedQueueRpcExecutor$FastPathHandler.getCallRunner(FastPathBalancedQueueRpcExecutor.java:109) at org.apache.hadoop.hbase.ipc.RpcExecutor$Handler.run(RpcExecutor.java:334) Locked ownable synchronizers: - None "RpcServer.priority.RWQ.Fifo.read.handler=9,queue=1,port=16201" #59 daemon prio=5 os_prio=0 tid=0x00007fcd264a2000 nid=0x6b93 waiting on condition [0x00007fccf736d000] java.lang.Thread.State: WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00000000a01578f0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) at org.apache.hadoop.hbase.ipc.RpcExecutor$Handler.getCallRunner(RpcExecutor.java:325) at org.apache.hadoop.hbase.ipc.RpcExecutor$Handler.run(RpcExecutor.java:334) Locked ownable synchronizers: - None "RpcServer.priority.RWQ.Fifo.read.handler=8,queue=1,port=16201" #58 daemon prio=5 os_prio=0 tid=0x00007fcd264a0000 nid=0x6b92 waiting on condition [0x00007fccf746e000] java.lang.Thread.State: WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00000000a01578f0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) at org.apache.hadoop.hbase.ipc.RpcExecutor$Handler.getCallRunner(RpcExecutor.java:325) at org.apache.hadoop.hbase.ipc.RpcExecutor$Handler.run(RpcExecutor.java:334) Locked ownable synchronizers: - None "RpcServer.priority.RWQ.Fifo.read.handler=7,queue=1,port=16201" #57 daemon prio=5 os_prio=0 tid=0x00007fcd2649e000 nid=0x6b91 waiting on condition [0x00007fccf756f000] java.lang.Thread.State: WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00000000a01578f0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) at org.apache.hadoop.hbase.ipc.RpcExecutor$Handler.getCallRunner(RpcExecutor.java:325) at org.apache.hadoop.hbase.ipc.RpcExecutor$Handler.run(RpcExecutor.java:334) Locked ownable synchronizers: - None "RpcServer.priority.RWQ.Fifo.read.handler=6,queue=1,port=16201" #56 daemon prio=5 os_prio=0 tid=0x00007fcd264f7800 nid=0x6b90 waiting on condition [0x00007fccf7670000] java.lang.Thread.State: WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00000000a01578f0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) at org.apache.hadoop.hbase.ipc.RpcExecutor$Handler.getCallRunner(RpcExecutor.java:325) at org.apache.hadoop.hbase.ipc.RpcExecutor$Handler.run(RpcExecutor.java:334) Locked ownable synchronizers: - None "RpcServer.priority.RWQ.Fifo.read.handler=5,queue=1,port=16201" #55 daemon prio=5 os_prio=0 tid=0x00007fcd264f5800 nid=0x6b8f waiting on condition [0x00007fccf7771000] java.lang.Thread.State: WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00000000a01578f0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) at org.apache.hadoop.hbase.ipc.RpcExecutor$Handler.getCallRunner(RpcExecutor.java:325) at org.apache.hadoop.hbase.ipc.RpcExecutor$Handler.run(RpcExecutor.java:334) Locked ownable synchronizers: - None "RpcServer.priority.RWQ.Fifo.read.handler=4,queue=1,port=16201" #54 daemon prio=5 os_prio=0 tid=0x00007fcd264f3000 nid=0x6b8e waiting on condition [0x00007fccf7872000] java.lang.Thread.State: WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00000000a01578f0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) at org.apache.hadoop.hbase.ipc.RpcExecutor$Handler.getCallRunner(RpcExecutor.java:325) at org.apache.hadoop.hbase.ipc.RpcExecutor$Handler.run(RpcExecutor.java:334) Locked ownable synchronizers: - None "RpcServer.priority.RWQ.Fifo.read.handler=3,queue=1,port=16201" #53 daemon prio=5 os_prio=0 tid=0x00007fcd264f1000 nid=0x6b8d waiting on condition [0x00007fccf7973000] java.lang.Thread.State: WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00000000a01578f0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) at org.apache.hadoop.hbase.ipc.RpcExecutor$Handler.getCallRunner(RpcExecutor.java:325) at org.apache.hadoop.hbase.ipc.RpcExecutor$Handler.run(RpcExecutor.java:334) Locked ownable synchronizers: - None "RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=16201" #52 daemon prio=5 os_prio=0 tid=0x00007fcd264ef000 nid=0x6b8c waiting on condition [0x00007fccf7a74000] java.lang.Thread.State: WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00000000a01578f0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) at org.apache.hadoop.hbase.ipc.RpcExecutor$Handler.getCallRunner(RpcExecutor.java:325) at org.apache.hadoop.hbase.ipc.RpcExecutor$Handler.run(RpcExecutor.java:334) Locked ownable synchronizers: - None "RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=16201" #51 daemon prio=5 os_prio=0 tid=0x00007fcd264ed000 nid=0x6b8b waiting on condition [0x00007fccf7b75000] java.lang.Thread.State: WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00000000a01578f0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) at org.apache.hadoop.hbase.ipc.RpcExecutor$Handler.getCallRunner(RpcExecutor.java:325) at org.apache.hadoop.hbase.ipc.RpcExecutor$Handler.run(RpcExecutor.java:334) Locked ownable synchronizers: - None "RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16201" #50 daemon prio=5 os_prio=0 tid=0x00007fcd264eb000 nid=0x6b8a waiting on condition [0x00007fccf7c76000] java.lang.Thread.State: WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00000000a0157958> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) at org.apache.hadoop.hbase.ipc.RpcExecutor$Handler.getCallRunner(RpcExecutor.java:325) at org.apache.hadoop.hbase.ipc.RpcExecutor$Handler.run(RpcExecutor.java:334) Locked ownable synchronizers: - None "RpcServer.default.FPBQ.Fifo.handler=15,queue=1,port=16201" #49 daemon prio=5 os_prio=0 tid=0x00007fcd24350800 nid=0x6b89 waiting for monitor entry [0x00007fccf7d77000] java.lang.Thread.State: BLOCKED (on object monitor) at org.apache.hadoop.hbase.regionserver.MultiVersionConcurrencyControl.complete(MultiVersionConcurrencyControl.java:187) - waiting to lock <0x00000000a31b4eb8> (a java.util.LinkedList) at org.apache.hadoop.hbase.regionserver.HRegion.doWALAppend(HRegion.java:8706) at org.apache.hadoop.hbase.regionserver.HRegion.doMiniBatchMutate(HRegion.java:4687) at org.apache.hadoop.hbase.regionserver.HRegion.batchMutate(HRegion.java:4612) at org.apache.hadoop.hbase.regionserver.HRegion.batchMutate(HRegion.java:4541) at org.apache.hadoop.hbase.regionserver.RSRpcServices.doBatchOp(RSRpcServices.java:1020) at org.apache.hadoop.hbase.regionserver.RSRpcServices.doNonAtomicBatchOp(RSRpcServices.java:933) at org.apache.hadoop.hbase.regionserver.RSRpcServices.doNonAtomicRegionMutation(RSRpcServices.java:897) at org.apache.hadoop.hbase.regionserver.RSRpcServices.multi(RSRpcServices.java:2899) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45265) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:392) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:133) at org.apache.hadoop.hbase.ipc.RpcExecutor$Handler.run(RpcExecutor.java:354) at org.apache.hadoop.hbase.ipc.RpcExecutor$Handler.run(RpcExecutor.java:334) Locked ownable synchronizers: - None "RpcServer.default.FPBQ.Fifo.handler=14,queue=0,port=16201" #48 daemon prio=5 os_prio=0 tid=0x00007fcd2434e800 nid=0x6b88 waiting for monitor entry [0x00007fccf7e78000] java.lang.Thread.State: BLOCKED (on object monitor) at org.apache.hadoop.hbase.regionserver.MultiVersionConcurrencyControl.begin(MultiVersionConcurrencyControl.java:142) - waiting to lock <0x00000000a3105570> (a java.util.LinkedList) at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.stampSequenceIdAndPublishToRingBuffer(AbstractFSWAL.java:1085) at org.apache.hadoop.hbase.regionserver.wal.AsyncFSWAL.append(AsyncFSWAL.java:607) at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendData(AbstractFSWAL.java:1139) at org.apache.hadoop.hbase.regionserver.HRegion.doWALAppend(HRegion.java:8698) at org.apache.hadoop.hbase.regionserver.HRegion.doMiniBatchMutate(HRegion.java:4687) at org.apache.hadoop.hbase.regionserver.HRegion.batchMutate(HRegion.java:4612) at org.apache.hadoop.hbase.regionserver.HRegion.batchMutate(HRegion.java:4541) at org.apache.hadoop.hbase.regionserver.RSRpcServices.doBatchOp(RSRpcServices.java:1020) at org.apache.hadoop.hbase.regionserver.RSRpcServices.doNonAtomicBatchOp(RSRpcServices.java:933) at org.apache.hadoop.hbase.regionserver.RSRpcServices.doNonAtomicRegionMutation(RSRpcServices.java:897) at org.apache.hadoop.hbase.regionserver.RSRpcServices.multi(RSRpcServices.java:2899) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45265) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:392) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:133) at org.apache.hadoop.hbase.ipc.RpcExecutor$Handler.run(RpcExecutor.java:354) at org.apache.hadoop.hbase.ipc.RpcExecutor$Handler.run(RpcExecutor.java:334) Locked ownable synchronizers: - None "RpcServer.default.FPBQ.Fifo.handler=13,queue=1,port=16201" #47 daemon prio=5 os_prio=0 tid=0x00007fcd2434c800 nid=0x6b87 runnable [0x00007fccf7f79000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:338) at com.lmax.disruptor.MultiProducerSequencer.next(MultiProducerSequencer.java:136) at com.lmax.disruptor.MultiProducerSequencer.next(MultiProducerSequencer.java:105) at com.lmax.disruptor.RingBuffer.next(RingBuffer.java:263) at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$stampSequenceIdAndPublishToRingBuffer$2(AbstractFSWAL.java:1086) at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL$$Lambda$179/1460879368.run(Unknown Source) at org.apache.hadoop.hbase.regionserver.MultiVersionConcurrencyControl.begin(MultiVersionConcurrencyControl.java:145) - locked <0x00000000a30dc890> (a java.util.LinkedList) at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.stampSequenceIdAndPublishToRingBuffer(AbstractFSWAL.java:1085) at org.apache.hadoop.hbase.regionserver.wal.AsyncFSWAL.append(AsyncFSWAL.java:607) at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendData(AbstractFSWAL.java:1139) at org.apache.hadoop.hbase.regionserver.HRegion.doWALAppend(HRegion.java:8698) at org.apache.hadoop.hbase.regionserver.HRegion.doMiniBatchMutate(HRegion.java:4687) at org.apache.hadoop.hbase.regionserver.HRegion.batchMutate(HRegion.java:4612) at org.apache.hadoop.hbase.regionserver.HRegion.batchMutate(HRegion.java:4541) at org.apache.hadoop.hbase.regionserver.RSRpcServices.doBatchOp(RSRpcServices.java:1020) at org.apache.hadoop.hbase.regionserver.RSRpcServices.doNonAtomicBatchOp(RSRpcServices.java:933) at org.apache.hadoop.hbase.regionserver.RSRpcServices.doNonAtomicRegionMutation(RSRpcServices.java:897) at org.apache.hadoop.hbase.regionserver.RSRpcServices.multi(RSRpcServices.java:2899) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45265) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:392) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:133) at org.apache.hadoop.hbase.ipc.RpcExecutor$Handler.run(RpcExecutor.java:354) at org.apache.hadoop.hbase.ipc.RpcExecutor$Handler.run(RpcExecutor.java:334) Locked ownable synchronizers: - None "RpcServer.default.FPBQ.Fifo.handler=12,queue=0,port=16201" #46 daemon prio=5 os_prio=0 tid=0x00007fcd2434a800 nid=0x6b86 waiting for monitor entry [0x00007fccf807a000] java.lang.Thread.State: BLOCKED (on object monitor) at org.apache.hadoop.hbase.regionserver.MultiVersionConcurrencyControl.begin(MultiVersionConcurrencyControl.java:142) - waiting to lock <0x00000000a31b4eb8> (a java.util.LinkedList) at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.stampSequenceIdAndPublishToRingBuffer(AbstractFSWAL.java:1085) at org.apache.hadoop.hbase.regionserver.wal.AsyncFSWAL.append(AsyncFSWAL.java:607) at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendData(AbstractFSWAL.java:1139) at org.apache.hadoop.hbase.regionserver.HRegion.doWALAppend(HRegion.java:8698) at org.apache.hadoop.hbase.regionserver.HRegion.doMiniBatchMutate(HRegion.java:4687) at org.apache.hadoop.hbase.regionserver.HRegion.batchMutate(HRegion.java:4612) at org.apache.hadoop.hbase.regionserver.HRegion.batchMutate(HRegion.java:4541) at org.apache.hadoop.hbase.regionserver.RSRpcServices.doBatchOp(RSRpcServices.java:1020) at org.apache.hadoop.hbase.regionserver.RSRpcServices.doNonAtomicBatchOp(RSRpcServices.java:933) at org.apache.hadoop.hbase.regionserver.RSRpcServices.doNonAtomicRegionMutation(RSRpcServices.java:897) at org.apache.hadoop.hbase.regionserver.RSRpcServices.multi(RSRpcServices.java:2899) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45265) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:392) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:133) at org.apache.hadoop.hbase.ipc.RpcExecutor$Handler.run(RpcExecutor.java:354) at org.apache.hadoop.hbase.ipc.RpcExecutor$Handler.run(RpcExecutor.java:334) Locked ownable synchronizers: - None "RpcServer.default.FPBQ.Fifo.handler=11,queue=1,port=16201" #45 daemon prio=5 os_prio=0 tid=0x00007fcd266e3000 nid=0x6b85 waiting for monitor entry [0x00007fccf817b000] java.lang.Thread.State: BLOCKED (on object monitor) at org.apache.hadoop.hbase.regionserver.MultiVersionConcurrencyControl.complete(MultiVersionConcurrencyControl.java:187) - waiting to lock <0x00000000a31b4eb8> (a java.util.LinkedList) at org.apache.hadoop.hbase.regionserver.HRegion.doWALAppend(HRegion.java:8706) at org.apache.hadoop.hbase.regionserver.HRegion.doMiniBatchMutate(HRegion.java:4687) at org.apache.hadoop.hbase.regionserver.HRegion.batchMutate(HRegion.java:4612) at org.apache.hadoop.hbase.regionserver.HRegion.batchMutate(HRegion.java:4541) at org.apache.hadoop.hbase.regionserver.RSRpcServices.doBatchOp(RSRpcServices.java:1020) at org.apache.hadoop.hbase.regionserver.RSRpcServices.doNonAtomicBatchOp(RSRpcServices.java:933) at org.apache.hadoop.hbase.regionserver.RSRpcServices.doNonAtomicRegionMutation(RSRpcServices.java:897) at org.apache.hadoop.hbase.regionserver.RSRpcServices.multi(RSRpcServices.java:2899) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45265) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:392) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:133) at org.apache.hadoop.hbase.ipc.RpcExecutor$Handler.run(RpcExecutor.java:354) at org.apache.hadoop.hbase.ipc.RpcExecutor$Handler.run(RpcExecutor.java:334) Locked ownable synchronizers: - None "RpcServer.default.FPBQ.Fifo.handler=10,queue=0,port=16201" #44 daemon prio=5 os_prio=0 tid=0x00007fcd266e1000 nid=0x6b84 runnable [0x00007fccf827c000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:338) at com.lmax.disruptor.MultiProducerSequencer.next(MultiProducerSequencer.java:136) at com.lmax.disruptor.MultiProducerSequencer.next(MultiProducerSequencer.java:105) at com.lmax.disruptor.RingBuffer.next(RingBuffer.java:263) at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$stampSequenceIdAndPublishToRingBuffer$2(AbstractFSWAL.java:1086) at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL$$Lambda$179/1460879368.run(Unknown Source) at org.apache.hadoop.hbase.regionserver.MultiVersionConcurrencyControl.begin(MultiVersionConcurrencyControl.java:145) - locked <0x00000000a31b4eb8> (a java.util.LinkedList) at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.stampSequenceIdAndPublishToRingBuffer(AbstractFSWAL.java:1085) at org.apache.hadoop.hbase.regionserver.wal.AsyncFSWAL.append(AsyncFSWAL.java:607) at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendData(AbstractFSWAL.java:1139) at org.apache.hadoop.hbase.regionserver.HRegion.doWALAppend(HRegion.java:8698) at org.apache.hadoop.hbase.regionserver.HRegion.doMiniBatchMutate(HRegion.java:4687) at org.apache.hadoop.hbase.regionserver.HRegion.batchMutate(HRegion.java:4612) at org.apache.hadoop.hbase.regionserver.HRegion.batchMutate(HRegion.java:4541) at org.apache.hadoop.hbase.regionserver.RSRpcServices.doBatchOp(RSRpcServices.java:1020) at org.apache.hadoop.hbase.regionserver.RSRpcServices.doNonAtomicBatchOp(RSRpcServices.java:933) at org.apache.hadoop.hbase.regionserver.RSRpcServices.doNonAtomicRegionMutation(RSRpcServices.java:897) at org.apache.hadoop.hbase.regionserver.RSRpcServices.multi(RSRpcServices.java:2899) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45265) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:392) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:133) at org.apache.hadoop.hbase.ipc.RpcExecutor$Handler.run(RpcExecutor.java:354) at org.apache.hadoop.hbase.ipc.RpcExecutor$Handler.run(RpcExecutor.java:334) Locked ownable synchronizers: - None "RpcServer.default.FPBQ.Fifo.handler=9,queue=1,port=16201" #43 daemon prio=5 os_prio=0 tid=0x00007fcd266de800 nid=0x6b83 waiting for monitor entry [0x00007fccf837d000] java.lang.Thread.State: BLOCKED (on object monitor) at org.apache.hadoop.hbase.regionserver.MultiVersionConcurrencyControl.complete(MultiVersionConcurrencyControl.java:187) - waiting to lock <0x00000000a31bf6b0> (a java.util.LinkedList) at org.apache.hadoop.hbase.regionserver.HRegion.doWALAppend(HRegion.java:8706) at org.apache.hadoop.hbase.regionserver.HRegion.doMiniBatchMutate(HRegion.java:4687) at org.apache.hadoop.hbase.regionserver.HRegion.batchMutate(HRegion.java:4612) at org.apache.hadoop.hbase.regionserver.HRegion.batchMutate(HRegion.java:4541) at org.apache.hadoop.hbase.regionserver.RSRpcServices.doBatchOp(RSRpcServices.java:1020) at org.apache.hadoop.hbase.regionserver.RSRpcServices.doNonAtomicBatchOp(RSRpcServices.java:933) at org.apache.hadoop.hbase.regionserver.RSRpcServices.doNonAtomicRegionMutation(RSRpcServices.java:897) at org.apache.hadoop.hbase.regionserver.RSRpcServices.multi(RSRpcServices.java:2899) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45265) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:392) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:133) at org.apache.hadoop.hbase.ipc.RpcExecutor$Handler.run(RpcExecutor.java:354) at org.apache.hadoop.hbase.ipc.RpcExecutor$Handler.run(RpcExecutor.java:334) Locked ownable synchronizers: - None "RpcServer.default.FPBQ.Fifo.handler=8,queue=0,port=16201" #42 daemon prio=5 os_prio=0 tid=0x00007fcd266dd000 nid=0x6b82 runnable [0x00007fccf847d000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:338) at com.lmax.disruptor.MultiProducerSequencer.next(MultiProducerSequencer.java:136) at com.lmax.disruptor.MultiProducerSequencer.next(MultiProducerSequencer.java:105) at com.lmax.disruptor.RingBuffer.next(RingBuffer.java:263) at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$stampSequenceIdAndPublishToRingBuffer$2(AbstractFSWAL.java:1086) at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL$$Lambda$179/1460879368.run(Unknown Source) at org.apache.hadoop.hbase.regionserver.MultiVersionConcurrencyControl.begin(MultiVersionConcurrencyControl.java:145) - locked <0x00000000a31bf6b0> (a java.util.LinkedList) at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.stampSequenceIdAndPublishToRingBuffer(AbstractFSWAL.java:1085) at org.apache.hadoop.hbase.regionserver.wal.AsyncFSWAL.append(AsyncFSWAL.java:607) at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendData(AbstractFSWAL.java:1139) at org.apache.hadoop.hbase.regionserver.HRegion.doWALAppend(HRegion.java:8698) at org.apache.hadoop.hbase.regionserver.HRegion.doMiniBatchMutate(HRegion.java:4687) at org.apache.hadoop.hbase.regionserver.HRegion.batchMutate(HRegion.java:4612) at org.apache.hadoop.hbase.regionserver.HRegion.batchMutate(HRegion.java:4541) at org.apache.hadoop.hbase.regionserver.RSRpcServices.doBatchOp(RSRpcServices.java:1020) at org.apache.hadoop.hbase.regionserver.RSRpcServices.doNonAtomicBatchOp(RSRpcServices.java:933) at org.apache.hadoop.hbase.regionserver.RSRpcServices.doNonAtomicRegionMutation(RSRpcServices.java:897) at org.apache.hadoop.hbase.regionserver.RSRpcServices.multi(RSRpcServices.java:2899) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45265) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:392) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:133) at org.apache.hadoop.hbase.ipc.RpcExecutor$Handler.run(RpcExecutor.java:354) at org.apache.hadoop.hbase.ipc.RpcExecutor$Handler.run(RpcExecutor.java:334) Locked ownable synchronizers: - None "RpcServer.default.FPBQ.Fifo.handler=7,queue=1,port=16201" #41 daemon prio=5 os_prio=0 tid=0x00007fcd266ba000 nid=0x6b81 waiting for monitor entry [0x00007fccf857f000] java.lang.Thread.State: BLOCKED (on object monitor) at org.apache.hadoop.hbase.regionserver.MultiVersionConcurrencyControl.complete(MultiVersionConcurrencyControl.java:187) - waiting to lock <0x00000000a3105570> (a java.util.LinkedList) at org.apache.hadoop.hbase.regionserver.HRegion.doWALAppend(HRegion.java:8706) at org.apache.hadoop.hbase.regionserver.HRegion.doMiniBatchMutate(HRegion.java:4687) at org.apache.hadoop.hbase.regionserver.HRegion.batchMutate(HRegion.java:4612) at org.apache.hadoop.hbase.regionserver.HRegion.batchMutate(HRegion.java:4541) at org.apache.hadoop.hbase.regionserver.RSRpcServices.doBatchOp(RSRpcServices.java:1020) at org.apache.hadoop.hbase.regionserver.RSRpcServices.doNonAtomicBatchOp(RSRpcServices.java:933) at org.apache.hadoop.hbase.regionserver.RSRpcServices.doNonAtomicRegionMutation(RSRpcServices.java:897) at org.apache.hadoop.hbase.regionserver.RSRpcServices.multi(RSRpcServices.java:2899) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45265) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:392) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:133) at org.apache.hadoop.hbase.ipc.RpcExecutor$Handler.run(RpcExecutor.java:354) at org.apache.hadoop.hbase.ipc.RpcExecutor$Handler.run(RpcExecutor.java:334) Locked ownable synchronizers: - None "RpcServer.default.FPBQ.Fifo.handler=6,queue=0,port=16201" #40 daemon prio=5 os_prio=0 tid=0x00007fcd266b8800 nid=0x6b80 waiting for monitor entry [0x00007fccf8680000] java.lang.Thread.State: BLOCKED (on object monitor) at org.apache.hadoop.hbase.regionserver.MultiVersionConcurrencyControl.begin(MultiVersionConcurrencyControl.java:142) - waiting to lock <0x00000000a31bf6b0> (a java.util.LinkedList) at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.stampSequenceIdAndPublishToRingBuffer(AbstractFSWAL.java:1085) at org.apache.hadoop.hbase.regionserver.wal.AsyncFSWAL.append(AsyncFSWAL.java:607) at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendData(AbstractFSWAL.java:1139) at org.apache.hadoop.hbase.regionserver.HRegion.doWALAppend(HRegion.java:8698) at org.apache.hadoop.hbase.regionserver.HRegion.doMiniBatchMutate(HRegion.java:4687) at org.apache.hadoop.hbase.regionserver.HRegion.batchMutate(HRegion.java:4612) at org.apache.hadoop.hbase.regionserver.HRegion.batchMutate(HRegion.java:4541) at org.apache.hadoop.hbase.regionserver.RSRpcServices.doBatchOp(RSRpcServices.java:1020) at org.apache.hadoop.hbase.regionserver.RSRpcServices.doNonAtomicBatchOp(RSRpcServices.java:933) at org.apache.hadoop.hbase.regionserver.RSRpcServices.doNonAtomicRegionMutation(RSRpcServices.java:897) at org.apache.hadoop.hbase.regionserver.RSRpcServices.multi(RSRpcServices.java:2899) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45265) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:392) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:133) at org.apache.hadoop.hbase.ipc.RpcExecutor$Handler.run(RpcExecutor.java:354) at org.apache.hadoop.hbase.ipc.RpcExecutor$Handler.run(RpcExecutor.java:334) Locked ownable synchronizers: - None "RpcServer.default.FPBQ.Fifo.handler=5,queue=1,port=16201" #39 daemon prio=5 os_prio=0 tid=0x00007fcd266b6800 nid=0x6b7f runnable [0x00007fccf8781000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:338) at com.lmax.disruptor.MultiProducerSequencer.next(MultiProducerSequencer.java:136) at com.lmax.disruptor.MultiProducerSequencer.next(MultiProducerSequencer.java:105) at com.lmax.disruptor.RingBuffer.next(RingBuffer.java:263) at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$stampSequenceIdAndPublishToRingBuffer$2(AbstractFSWAL.java:1086) at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL$$Lambda$179/1460879368.run(Unknown Source) at org.apache.hadoop.hbase.regionserver.MultiVersionConcurrencyControl.begin(MultiVersionConcurrencyControl.java:145) - locked <0x00000000a3105570> (a java.util.LinkedList) at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.stampSequenceIdAndPublishToRingBuffer(AbstractFSWAL.java:1085) at org.apache.hadoop.hbase.regionserver.wal.AsyncFSWAL.append(AsyncFSWAL.java:607) at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendData(AbstractFSWAL.java:1139) at org.apache.hadoop.hbase.regionserver.HRegion.doWALAppend(HRegion.java:8698) at org.apache.hadoop.hbase.regionserver.HRegion.doMiniBatchMutate(HRegion.java:4687) at org.apache.hadoop.hbase.regionserver.HRegion.batchMutate(HRegion.java:4612) at org.apache.hadoop.hbase.regionserver.HRegion.batchMutate(HRegion.java:4541) at org.apache.hadoop.hbase.regionserver.RSRpcServices.doBatchOp(RSRpcServices.java:1020) at org.apache.hadoop.hbase.regionserver.RSRpcServices.doNonAtomicBatchOp(RSRpcServices.java:933) at org.apache.hadoop.hbase.regionserver.RSRpcServices.doNonAtomicRegionMutation(RSRpcServices.java:897) at org.apache.hadoop.hbase.regionserver.RSRpcServices.multi(RSRpcServices.java:2899) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45265) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:392) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:133) at org.apache.hadoop.hbase.ipc.RpcExecutor$Handler.run(RpcExecutor.java:354) at org.apache.hadoop.hbase.ipc.RpcExecutor$Handler.run(RpcExecutor.java:334) Locked ownable synchronizers: - None "RpcServer.default.FPBQ.Fifo.handler=4,queue=0,port=16201" #38 daemon prio=5 os_prio=0 tid=0x00007fcd266b4800 nid=0x6b7e waiting for monitor entry [0x00007fccf8882000] java.lang.Thread.State: BLOCKED (on object monitor) at org.apache.hadoop.hbase.regionserver.MultiVersionConcurrencyControl.begin(MultiVersionConcurrencyControl.java:142) - waiting to lock <0x00000000a3105570> (a java.util.LinkedList) at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.stampSequenceIdAndPublishToRingBuffer(AbstractFSWAL.java:1085) at org.apache.hadoop.hbase.regionserver.wal.AsyncFSWAL.append(AsyncFSWAL.java:607) at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendData(AbstractFSWAL.java:1139) at org.apache.hadoop.hbase.regionserver.HRegion.doWALAppend(HRegion.java:8698) at org.apache.hadoop.hbase.regionserver.HRegion.doMiniBatchMutate(HRegion.java:4687) at org.apache.hadoop.hbase.regionserver.HRegion.batchMutate(HRegion.java:4612) at org.apache.hadoop.hbase.regionserver.HRegion.batchMutate(HRegion.java:4541) at org.apache.hadoop.hbase.regionserver.RSRpcServices.doBatchOp(RSRpcServices.java:1020) at org.apache.hadoop.hbase.regionserver.RSRpcServices.doNonAtomicBatchOp(RSRpcServices.java:933) at org.apache.hadoop.hbase.regionserver.RSRpcServices.doNonAtomicRegionMutation(RSRpcServices.java:897) at org.apache.hadoop.hbase.regionserver.RSRpcServices.multi(RSRpcServices.java:2899) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45265) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:392) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:133) at org.apache.hadoop.hbase.ipc.RpcExecutor$Handler.run(RpcExecutor.java:354) at org.apache.hadoop.hbase.ipc.RpcExecutor$Handler.run(RpcExecutor.java:334) Locked ownable synchronizers: - None "RpcServer.default.FPBQ.Fifo.handler=3,queue=1,port=16201" #37 daemon prio=5 os_prio=0 tid=0x00007fcd266b3800 nid=0x6b7d runnable [0x00007fccf8982000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:338) at com.lmax.disruptor.MultiProducerSequencer.next(MultiProducerSequencer.java:136) at com.lmax.disruptor.MultiProducerSequencer.next(MultiProducerSequencer.java:105) at com.lmax.disruptor.RingBuffer.next(RingBuffer.java:263) at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$stampSequenceIdAndPublishToRingBuffer$2(AbstractFSWAL.java:1086) at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL$$Lambda$179/1460879368.run(Unknown Source) at org.apache.hadoop.hbase.regionserver.MultiVersionConcurrencyControl.begin(MultiVersionConcurrencyControl.java:145) - locked <0x00000000a31c14b8> (a java.util.LinkedList) at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.stampSequenceIdAndPublishToRingBuffer(AbstractFSWAL.java:1085) at org.apache.hadoop.hbase.regionserver.wal.AsyncFSWAL.append(AsyncFSWAL.java:607) at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendData(AbstractFSWAL.java:1139) at org.apache.hadoop.hbase.regionserver.HRegion.doWALAppend(HRegion.java:8698) at org.apache.hadoop.hbase.regionserver.HRegion.doMiniBatchMutate(HRegion.java:4687) at org.apache.hadoop.hbase.regionserver.HRegion.batchMutate(HRegion.java:4612) at org.apache.hadoop.hbase.regionserver.HRegion.batchMutate(HRegion.java:4541) at org.apache.hadoop.hbase.regionserver.RSRpcServices.doBatchOp(RSRpcServices.java:1020) at org.apache.hadoop.hbase.regionserver.RSRpcServices.doNonAtomicBatchOp(RSRpcServices.java:933) at org.apache.hadoop.hbase.regionserver.RSRpcServices.doNonAtomicRegionMutation(RSRpcServices.java:897) at org.apache.hadoop.hbase.regionserver.RSRpcServices.multi(RSRpcServices.java:2899) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45265) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:392) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:133) at org.apache.hadoop.hbase.ipc.RpcExecutor$Handler.run(RpcExecutor.java:354) at org.apache.hadoop.hbase.ipc.RpcExecutor$Handler.run(RpcExecutor.java:334) Locked ownable synchronizers: - None "RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=16201" #36 daemon prio=5 os_prio=0 tid=0x00007fcd27003000 nid=0x6b7c waiting for monitor entry [0x00007fccf8a84000] java.lang.Thread.State: BLOCKED (on object monitor) at org.apache.hadoop.hbase.regionserver.MultiVersionConcurrencyControl.begin(MultiVersionConcurrencyControl.java:142) - waiting to lock <0x00000000a3105570> (a java.util.LinkedList) at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.stampSequenceIdAndPublishToRingBuffer(AbstractFSWAL.java:1085) at org.apache.hadoop.hbase.regionserver.wal.AsyncFSWAL.append(AsyncFSWAL.java:607) at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendData(AbstractFSWAL.java:1139) at org.apache.hadoop.hbase.regionserver.HRegion.doWALAppend(HRegion.java:8698) at org.apache.hadoop.hbase.regionserver.HRegion.doMiniBatchMutate(HRegion.java:4687) at org.apache.hadoop.hbase.regionserver.HRegion.batchMutate(HRegion.java:4612) at org.apache.hadoop.hbase.regionserver.HRegion.batchMutate(HRegion.java:4541) at org.apache.hadoop.hbase.regionserver.RSRpcServices.doBatchOp(RSRpcServices.java:1020) at org.apache.hadoop.hbase.regionserver.RSRpcServices.doNonAtomicBatchOp(RSRpcServices.java:933) at org.apache.hadoop.hbase.regionserver.RSRpcServices.doNonAtomicRegionMutation(RSRpcServices.java:897) at org.apache.hadoop.hbase.regionserver.RSRpcServices.multi(RSRpcServices.java:2899) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45265) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:392) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:133) at org.apache.hadoop.hbase.ipc.RpcExecutor$Handler.run(RpcExecutor.java:354) at org.apache.hadoop.hbase.ipc.RpcExecutor$Handler.run(RpcExecutor.java:334) Locked ownable synchronizers: - None "RpcServer.default.FPBQ.Fifo.handler=1,queue=1,port=16201" #35 daemon prio=5 os_prio=0 tid=0x00007fcd26ffb800 nid=0x6b7b waiting for monitor entry [0x00007fccf8b85000] java.lang.Thread.State: BLOCKED (on object monitor) at org.apache.hadoop.hbase.regionserver.MultiVersionConcurrencyControl.complete(MultiVersionConcurrencyControl.java:187) - waiting to lock <0x00000000a31bf6b0> (a java.util.LinkedList) at org.apache.hadoop.hbase.regionserver.HRegion.doWALAppend(HRegion.java:8706) at org.apache.hadoop.hbase.regionserver.HRegion.doMiniBatchMutate(HRegion.java:4687) at org.apache.hadoop.hbase.regionserver.HRegion.batchMutate(HRegion.java:4612) at org.apache.hadoop.hbase.regionserver.HRegion.batchMutate(HRegion.java:4541) at org.apache.hadoop.hbase.regionserver.RSRpcServices.doBatchOp(RSRpcServices.java:1020) at org.apache.hadoop.hbase.regionserver.RSRpcServices.doNonAtomicBatchOp(RSRpcServices.java:933) at org.apache.hadoop.hbase.regionserver.RSRpcServices.doNonAtomicRegionMutation(RSRpcServices.java:897) at org.apache.hadoop.hbase.regionserver.RSRpcServices.multi(RSRpcServices.java:2899) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45265) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:392) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:133) at org.apache.hadoop.hbase.ipc.RpcExecutor$Handler.run(RpcExecutor.java:354) at org.apache.hadoop.hbase.ipc.RpcExecutor$Handler.run(RpcExecutor.java:334) Locked ownable synchronizers: - None "RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=16201" #34 daemon prio=5 os_prio=0 tid=0x00007fcd26ffa000 nid=0x6b7a waiting for monitor entry [0x00007fccf8c86000] java.lang.Thread.State: BLOCKED (on object monitor) at org.apache.hadoop.hbase.regionserver.MultiVersionConcurrencyControl.begin(MultiVersionConcurrencyControl.java:142) - waiting to lock <0x00000000a31c14b8> (a java.util.LinkedList) at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.stampSequenceIdAndPublishToRingBuffer(AbstractFSWAL.java:1085) at org.apache.hadoop.hbase.regionserver.wal.AsyncFSWAL.append(AsyncFSWAL.java:607) at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendData(AbstractFSWAL.java:1139) at org.apache.hadoop.hbase.regionserver.HRegion.doWALAppend(HRegion.java:8698) at org.apache.hadoop.hbase.regionserver.HRegion.doMiniBatchMutate(HRegion.java:4687) at org.apache.hadoop.hbase.regionserver.HRegion.batchMutate(HRegion.java:4612) at org.apache.hadoop.hbase.regionserver.HRegion.batchMutate(HRegion.java:4541) at org.apache.hadoop.hbase.regionserver.RSRpcServices.doBatchOp(RSRpcServices.java:1020) at org.apache.hadoop.hbase.regionserver.RSRpcServices.doNonAtomicBatchOp(RSRpcServices.java:933) at org.apache.hadoop.hbase.regionserver.RSRpcServices.doNonAtomicRegionMutation(RSRpcServices.java:897) at org.apache.hadoop.hbase.regionserver.RSRpcServices.multi(RSRpcServices.java:2899) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45265) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:392) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:133) at org.apache.hadoop.hbase.ipc.RpcExecutor$Handler.run(RpcExecutor.java:354) at org.apache.hadoop.hbase.ipc.RpcExecutor$Handler.run(RpcExecutor.java:334) Locked ownable synchronizers: - None "zk-event-processor-pool-0" #33 daemon prio=5 os_prio=0 tid=0x00007fcd080c4000 nid=0x6b79 waiting on condition [0x00007fccf8d87000] java.lang.Thread.State: WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00000000a2403720> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "main-EventThread" #32 daemon prio=5 os_prio=0 tid=0x00007fcd271b7000 nid=0x6b78 waiting on condition [0x00007fccf8e88000] java.lang.Thread.State: WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00000000a2403998> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) at org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:506) Locked ownable synchronizers: - None "main-SendThread(cvp365.sjc.aristanetworks.com:2181)" #31 daemon prio=5 os_prio=0 tid=0x00007fcd271bb000 nid=0x6b77 runnable [0x00007fccf8f89000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00000000a2403c40> (a sun.nio.ch.Util$3) - locked <0x00000000a2403c30> (a java.util.Collections$UnmodifiableSet) - locked <0x00000000a2403be8> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.zookeeper.ClientCnxnSocketNIO.doTransport(ClientCnxnSocketNIO.java:345) at org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1223) Locked ownable synchronizers: - None "client DomainSocketWatcher" #28 daemon prio=5 os_prio=0 tid=0x00007fcd244f4800 nid=0x6b76 runnable [0x00007fccf908a000] java.lang.Thread.State: RUNNABLE at org.apache.hadoop.net.unix.DomainSocketWatcher.doPoll0(Native Method) at org.apache.hadoop.net.unix.DomainSocketWatcher.access$900(DomainSocketWatcher.java:52) at org.apache.hadoop.net.unix.DomainSocketWatcher$2.run(DomainSocketWatcher.java:503) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "RS-EventLoopGroup-1-2" #27 daemon prio=10 os_prio=0 tid=0x00007fcd08091800 nid=0x6b56 runnable [0x00007fccfa7e0000] java.lang.Thread.State: RUNNABLE at org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) at org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:192) at org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:185) at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:290) at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:347) at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:989) at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "org.apache.hadoop.fs.FileSystem$Statistics$StatisticsDataReferenceCleaner" #26 daemon prio=5 os_prio=0 tid=0x00007fcd262c5800 nid=0x6b4f in Object.wait() [0x00007fccfaae1000] java.lang.Thread.State: WAITING (on object monitor) at java.lang.Object.wait(Native Method) at java.lang.ref.ReferenceQueue.remove(ReferenceQueue.java:144) - locked <0x00000000a236d260> (a java.lang.ref.ReferenceQueue$Lock) at java.lang.ref.ReferenceQueue.remove(ReferenceQueue.java:165) at org.apache.hadoop.fs.FileSystem$Statistics$StatisticsDataReferenceCleaner.run(FileSystem.java:3760) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "MobFileCache #0" #25 daemon prio=5 os_prio=0 tid=0x00007fcd262bd800 nid=0x6b4d waiting on condition [0x00007fccfabe2000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00000000a24042d0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1093) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "LruBlockCacheStatsExecutor" #23 daemon prio=5 os_prio=0 tid=0x00007fcd262d8000 nid=0x6b4c waiting on condition [0x00007fccface3000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00000000a236d418> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1093) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "main.LruBlockCache.EvictionThread" #21 daemon prio=5 os_prio=0 tid=0x00007fcd262d9800 nid=0x6b4b in Object.wait() [0x00007fccfade4000] java.lang.Thread.State: TIMED_WAITING (on object monitor) at java.lang.Object.wait(Native Method) at org.apache.hadoop.hbase.io.hfile.LruBlockCache$EvictionThread.run(LruBlockCache.java:942) - locked <0x00000000a24043b8> (a org.apache.hadoop.hbase.io.hfile.LruBlockCache$EvictionThread) Locked ownable synchronizers: - None "RS-EventLoopGroup-1-1" #20 daemon prio=10 os_prio=0 tid=0x00007fcd261a2000 nid=0x6b4a runnable [0x00007fccfb0e7000] java.lang.Thread.State: RUNNABLE at org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) at org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:192) at org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:185) at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:290) at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:347) at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:989) at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "HBase-Metrics2-1" #19 daemon prio=5 os_prio=0 tid=0x00007fcd25dcc000 nid=0x6b27 waiting on condition [0x00007fccfb810000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00000000a01a03c0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1093) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "Service Thread" #14 daemon prio=9 os_prio=0 tid=0x00007fcd249ba000 nid=0x6acd runnable [0x0000000000000000] java.lang.Thread.State: RUNNABLE Locked ownable synchronizers: - None "C1 CompilerThread3" #13 daemon prio=9 os_prio=0 tid=0x00007fcd249b3800 nid=0x6acc waiting on condition [0x0000000000000000] java.lang.Thread.State: RUNNABLE Locked ownable synchronizers: - None "C2 CompilerThread2" #12 daemon prio=9 os_prio=0 tid=0x00007fcd2497b000 nid=0x6acb waiting on condition [0x0000000000000000] java.lang.Thread.State: RUNNABLE Locked ownable synchronizers: - None "C2 CompilerThread1" #11 daemon prio=9 os_prio=0 tid=0x00007fcd24979000 nid=0x6aca waiting on condition [0x0000000000000000] java.lang.Thread.State: RUNNABLE Locked ownable synchronizers: - None "C2 CompilerThread0" #10 daemon prio=9 os_prio=0 tid=0x00007fcd24978000 nid=0x6ac9 waiting on condition [0x0000000000000000] java.lang.Thread.State: RUNNABLE Locked ownable synchronizers: - None "Thread-2" #9 daemon prio=5 os_prio=0 tid=0x00007fcd08003000 nid=0x6ac8 runnable [0x00007fccfc966000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00000000a01986c8> (a sun.nio.ch.Util$3) - locked <0x00000000a01986b8> (a java.util.Collections$UnmodifiableSet) - locked <0x00000000a0198670> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at sun.net.httpserver.ServerImpl$Dispatcher.run(ServerImpl.java:352) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "server-timer" #7 daemon prio=5 os_prio=0 tid=0x00007fcd2498c800 nid=0x6ac6 in Object.wait() [0x00007fccfcb68000] java.lang.Thread.State: TIMED_WAITING (on object monitor) at java.lang.Object.wait(Native Method) at java.util.TimerThread.mainLoop(Timer.java:552) - locked <0x00000000a01abe50> (a java.util.TaskQueue) at java.util.TimerThread.run(Timer.java:505) Locked ownable synchronizers: - None "Signal Dispatcher" #5 daemon prio=9 os_prio=0 tid=0x00007fcd24270000 nid=0x69cc runnable [0x0000000000000000] java.lang.Thread.State: RUNNABLE Locked ownable synchronizers: - None "Surrogate Locker Thread (Concurrent GC)" #4 daemon prio=9 os_prio=0 tid=0x00007fcd2426e800 nid=0x69cb waiting on condition [0x0000000000000000] java.lang.Thread.State: RUNNABLE Locked ownable synchronizers: - None "Finalizer" #3 daemon prio=8 os_prio=0 tid=0x00007fcd24239000 nid=0x69ac in Object.wait() [0x00007fcd049e0000] java.lang.Thread.State: WAITING (on object monitor) at java.lang.Object.wait(Native Method) at java.lang.ref.ReferenceQueue.remove(ReferenceQueue.java:144) - locked <0x00000000a01abe40> (a java.lang.ref.ReferenceQueue$Lock) at java.lang.ref.ReferenceQueue.remove(ReferenceQueue.java:165) at java.lang.ref.Finalizer$FinalizerThread.run(Finalizer.java:216) Locked ownable synchronizers: - None "Reference Handler" #2 daemon prio=10 os_prio=0 tid=0x00007fcd24234800 nid=0x69ab in Object.wait() [0x00007fcd04ae1000] java.lang.Thread.State: WAITING (on object monitor) at java.lang.Object.wait(Native Method) at java.lang.Object.wait(Object.java:502) at java.lang.ref.Reference.tryHandlePending(Reference.java:191) - locked <0x00000000a01abe30> (a java.lang.ref.Reference$Lock) at java.lang.ref.Reference$ReferenceHandler.run(Reference.java:153) Locked ownable synchronizers: - None "main" #1 prio=5 os_prio=0 tid=0x00007fcd24016800 nid=0x698b in Object.wait() [0x00007fcd2b759000] java.lang.Thread.State: WAITING (on object monitor) at java.lang.Object.wait(Native Method) - waiting on <0x00000000a0156d08> (a org.apache.hadoop.hbase.regionserver.HRegionServer) at java.lang.Thread.join(Thread.java:1252) - locked <0x00000000a0156d08> (a org.apache.hadoop.hbase.regionserver.HRegionServer) at java.lang.Thread.join(Thread.java:1326) at org.apache.hadoop.hbase.regionserver.HRegionServerCommandLine.start(HRegionServerCommandLine.java:65) at org.apache.hadoop.hbase.regionserver.HRegionServerCommandLine.run(HRegionServerCommandLine.java:87) at org.apache.hadoop.util.ToolRunner.run(ToolRunner.java:76) at org.apache.hadoop.hbase.util.ServerCommandLine.doMain(ServerCommandLine.java:149) at org.apache.hadoop.hbase.regionserver.HRegionServer.main(HRegionServer.java:3184) Locked ownable synchronizers: - None "VM Thread" os_prio=0 tid=0x00007fcd2422a800 nid=0x69aa runnable "Gang worker#0 (Parallel GC Threads)" os_prio=0 tid=0x00007fcd2402b800 nid=0x698c runnable "Gang worker#1 (Parallel GC Threads)" os_prio=0 tid=0x00007fcd2402d800 nid=0x698d runnable "Gang worker#2 (Parallel GC Threads)" os_prio=0 tid=0x00007fcd2402f000 nid=0x698e runnable "Gang worker#3 (Parallel GC Threads)" os_prio=0 tid=0x00007fcd24031000 nid=0x698f runnable "Gang worker#4 (Parallel GC Threads)" os_prio=0 tid=0x00007fcd24033000 nid=0x6990 runnable "Gang worker#5 (Parallel GC Threads)" os_prio=0 tid=0x00007fcd24034800 nid=0x6991 runnable "Gang worker#6 (Parallel GC Threads)" os_prio=0 tid=0x00007fcd24036800 nid=0x6992 runnable "Gang worker#7 (Parallel GC Threads)" os_prio=0 tid=0x00007fcd24038800 nid=0x6993 runnable "G1 Main Concurrent Mark GC Thread" os_prio=0 tid=0x00007fcd24062000 nid=0x699d runnable "Gang worker#0 (G1 Parallel Marking Threads)" os_prio=0 tid=0x00007fcd24064000 nid=0x699e runnable "Gang worker#1 (G1 Parallel Marking Threads)" os_prio=0 tid=0x00007fcd24066000 nid=0x699f runnable "G1 Concurrent Refinement Thread#0" os_prio=0 tid=0x00007fcd2404a000 nid=0x699c runnable "G1 Concurrent Refinement Thread#1" os_prio=0 tid=0x00007fcd24048800 nid=0x699b runnable "G1 Concurrent Refinement Thread#2" os_prio=0 tid=0x00007fcd24046800 nid=0x699a runnable "G1 Concurrent Refinement Thread#3" os_prio=0 tid=0x00007fcd24044800 nid=0x6999 runnable "G1 Concurrent Refinement Thread#4" os_prio=0 tid=0x00007fcd24043000 nid=0x6998 runnable "G1 Concurrent Refinement Thread#5" os_prio=0 tid=0x00007fcd24041000 nid=0x6997 runnable "G1 Concurrent Refinement Thread#6" os_prio=0 tid=0x00007fcd2403f000 nid=0x6996 runnable "G1 Concurrent Refinement Thread#7" os_prio=0 tid=0x00007fcd2403d000 nid=0x6995 runnable "G1 Concurrent Refinement Thread#8" os_prio=0 tid=0x00007fcd2403b000 nid=0x6994 runnable "VM Periodic Task Thread" os_prio=0 tid=0x00007fcd256c6000 nid=0x6ace waiting on condition JNI global references: 894