Want more features on Pastebin? Sign Up, it's FREE!
0
0
Guest

RS stuck on WAL sync to a dead DN

By: a guest on Aug 26th, 2015  |  syntax: Java  |  size: 173.78 KB  |  views: 1  |  expires: in 24 hours
download  |  raw  |  embed  |  report abuse  |  print  |  QR code  |  clone
Text below is selected. Please press Ctrl+C to copy to your clipboard. (⌘+C on Mac)
  1. 2015-08-26 16:51:50
  2. Full thread dump OpenJDK 64-Bit Server VM (25.45-b02 mixed mode):
  3.  
  4. "Attach Listener" #33668 daemon prio=9 os_prio=0 tid=0x00007fbf552cf800 nid=0x5d9 waiting on condition [0x0000000000000000]
  5.    java.lang.Thread.State: RUNNABLE
  6.  
  7.    Locked ownable synchronizers:
  8.         - None
  9.  
  10. "RS_CLOSE_REGION-r12s16:9104-1" #33639 prio=5 os_prio=0 tid=0x00007fbf546fe000 nid=0x563 in Object.wait() [0x00007fbf38107000]
  11.    java.lang.Thread.State: WAITING (on object monitor)
  12.         at java.lang.Object.wait(Native Method)
  13.         at java.lang.Object.wait(Object.java:502)
  14.         at org.apache.hadoop.hbase.regionserver.HRegion.waitForFlushesAndCompactions(HRegion.java:1512)
  15.         - locked <0x000000056baa4888> (a org.apache.hadoop.hbase.regionserver.HRegion$WriteState)
  16.         at org.apache.hadoop.hbase.regionserver.HRegion.doClose(HRegion.java:1371)
  17.         - locked <0x000000056baa4888> (a org.apache.hadoop.hbase.regionserver.HRegion$WriteState)
  18.         at org.apache.hadoop.hbase.regionserver.HRegion.close(HRegion.java:1336)
  19.         - locked <0x000000056baaf928> (a java.lang.Object)
  20.         at org.apache.hadoop.hbase.regionserver.handler.CloseRegionHandler.process(CloseRegionHandler.java:138)
  21.         at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:128)
  22.         at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142)
  23.         at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
  24.         at java.lang.Thread.run(Thread.java:745)
  25.  
  26.    Locked ownable synchronizers:
  27.         - <0x000000054c683c18> (a java.util.concurrent.ThreadPoolExecutor$Worker)
  28.  
  29. "1243700389@qtp-864138492-3" #31589 daemon prio=5 os_prio=0 tid=0x0000000001f25000 nid=0x7b86 in Object.wait() [0x00007fbf3b3ae000]
  30.    java.lang.Thread.State: TIMED_WAITING (on object monitor)
  31.         at java.lang.Object.wait(Native Method)
  32.         at org.mortbay.thread.QueuedThreadPool$PoolThread.run(QueuedThreadPool.java:626)
  33.         - locked <0x00000005706cba60> (a org.mortbay.thread.QueuedThreadPool$PoolThread)
  34.  
  35.    Locked ownable synchronizers:
  36.         - None
  37.  
  38. "ResponseProcessor for block BP-1466258523-172.24.32.1-1437768622582:blk_1073817522_77115" #23856 daemon prio=5 os_prio=0 tid=0x00007fbf54163800 nid=0x5d51 runnable [0x00007fbf37d05000]
  39.    java.lang.Thread.State: RUNNABLE
  40.         at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method)
  41.         at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269)
  42.         at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:79)
  43.         at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86)
  44.         - locked <0x00000005530c94c0> (a sun.nio.ch.Util$2)
  45.         - locked <0x00000005530c97f0> (a java.util.Collections$UnmodifiableSet)
  46.         - locked <0x0000000548108aa8> (a sun.nio.ch.EPollSelectorImpl)
  47.         at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97)
  48.         at org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335)
  49.         at org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:157)
  50.         at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161)
  51.         at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131)
  52.         at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:118)
  53.         at java.io.FilterInputStream.read(FilterInputStream.java:83)
  54.         at java.io.FilterInputStream.read(FilterInputStream.java:83)
  55.         at org.apache.hadoop.hdfs.protocolPB.PBHelper.vintPrefixed(PBHelper.java:1998)
  56.         at org.apache.hadoop.hdfs.protocol.datatransfer.PipelineAck.readFields(PipelineAck.java:176)
  57.         at org.apache.hadoop.hdfs.DFSOutputStream$DataStreamer$ResponseProcessor.run(DFSOutputStream.java:798)
  58.  
  59.    Locked ownable synchronizers:
  60.         - None
  61.  
  62. "DataStreamer for file /hbase/WALs/r12s16.sjc.aristanetworks.com,9104,1439917659071/r12s16.sjc.aristanetworks.com%2C9104%2C1439917659071.default.1440314546058 block BP-1466258523-172.24.32.1-1437768622582:blk_1073817522_77115" #23855 daemon prio=5 os_prio=0 tid=0x00007fbf55a00800 nid=0x5d50 in Object.wait() [0x00007fbf3a317000]
  63.    java.lang.Thread.State: TIMED_WAITING (on object monitor)
  64.         at java.lang.Object.wait(Native Method)
  65.         at org.apache.hadoop.hdfs.DFSOutputStream$DataStreamer.run(DFSOutputStream.java:503)
  66.         - locked <0x000000054b9cd930> (a java.util.LinkedList)
  67.  
  68.    Locked ownable synchronizers:
  69.         - None
  70.  
  71. "RS_CLOSE_REGION-r12s16:9104-0" #10957 prio=5 os_prio=0 tid=0x00007fbf70c86800 nid=0x2af0 waiting on condition [0x00007fbf36ffa000]
  72.    java.lang.Thread.State: WAITING (parking)
  73.         at sun.misc.Unsafe.park(Native Method)
  74.         - parking to wait for  <0x00000005489a57e0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject)
  75.         at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)
  76.         at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039)
  77.         at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442)
  78.         at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1067)
  79.         at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127)
  80.         at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
  81.         at java.lang.Thread.run(Thread.java:745)
  82.  
  83.    Locked ownable synchronizers:
  84.         - None
  85.  
  86. "RS_LOG_REPLAY_OPS-r12s16:9104-1" #1991 prio=5 os_prio=0 tid=0x00007fbf54818800 nid=0x7ee waiting on condition [0x00007fbf3a519000]
  87.    java.lang.Thread.State: WAITING (parking)
  88.         at sun.misc.Unsafe.park(Native Method)
  89.         - parking to wait for  <0x00000005488cc0f8> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject)
  90.         at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)
  91.         at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039)
  92.         at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442)
  93.         at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1067)
  94.         at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127)
  95.         at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
  96.         at java.lang.Thread.run(Thread.java:745)
  97.  
  98.    Locked ownable synchronizers:
  99.         - None
  100.  
  101. "RS_LOG_REPLAY_OPS-r12s16:9104-0" #259 prio=5 os_prio=0 tid=0x00007fbf54214000 nid=0x12c waiting on condition [0x00007fbf3870b000]
  102.    java.lang.Thread.State: WAITING (parking)
  103.         at sun.misc.Unsafe.park(Native Method)
  104.         - parking to wait for  <0x00000005488cc0f8> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject)
  105.         at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)
  106.         at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039)
  107.         at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442)
  108.         at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1067)
  109.         at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127)
  110.         at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
  111.         at java.lang.Thread.run(Thread.java:745)
  112.  
  113.    Locked ownable synchronizers:
  114.         - None
  115.  
  116. "ReplicationExecutor-0" #258 daemon prio=5 os_prio=0 tid=0x00007fbf5c1eb800 nid=0x12b waiting on condition [0x00007fbf39d04000]
  117.    java.lang.Thread.State: WAITING (parking)
  118.         at sun.misc.Unsafe.park(Native Method)
  119.         - parking to wait for  <0x0000000548256b38> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject)
  120.         at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)
  121.         at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039)
  122.         at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442)
  123.         at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1067)
  124.         at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127)
  125.         at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
  126.         at java.lang.Thread.run(Thread.java:745)
  127.  
  128.    Locked ownable synchronizers:
  129.         - None
  130.  
  131. "ShortCircuitCache_SlotReleaser" #229 daemon prio=5 os_prio=0 tid=0x00007fbf5c0f6000 nid=0x10f waiting on condition [0x00007fbf38006000]
  132.    java.lang.Thread.State: WAITING (parking)
  133.         at sun.misc.Unsafe.park(Native Method)
  134.         - parking to wait for  <0x000000054860e1e0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject)
  135.         at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)
  136.         at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039)
  137.         at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1081)
  138.         at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809)
  139.         at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1067)
  140.         at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127)
  141.         at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
  142.         at java.lang.Thread.run(Thread.java:745)
  143.  
  144.    Locked ownable synchronizers:
  145.         - None
  146.  
  147. "RS_OPEN_REGION-r12s16:9104-2" #174 prio=5 os_prio=0 tid=0x00007fbf71772800 nid=0xd8 waiting on condition [0x00007fbf37800000]
  148.    java.lang.Thread.State: WAITING (parking)
  149.         at sun.misc.Unsafe.park(Native Method)
  150.         - parking to wait for  <0x00000005489d88d0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject)
  151.         at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)
  152.         at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039)
  153.         at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442)
  154.         at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1067)
  155.         at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127)
  156.         at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
  157.         at java.lang.Thread.run(Thread.java:745)
  158.  
  159.    Locked ownable synchronizers:
  160.         - None
  161.  
  162. "RS_OPEN_REGION-r12s16:9104-1" #166 prio=5 os_prio=0 tid=0x0000000001a8b800 nid=0xd0 waiting on condition [0x00007fbf37c04000]
  163.    java.lang.Thread.State: WAITING (parking)
  164.         at sun.misc.Unsafe.park(Native Method)
  165.         - parking to wait for  <0x00000005489d88d0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject)
  166.         at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)
  167.         at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039)
  168.         at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442)
  169.         at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1067)
  170.         at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127)
  171.         at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
  172.         at java.lang.Thread.run(Thread.java:745)
  173.  
  174.    Locked ownable synchronizers:
  175.         - None
  176.  
  177. "regionserver/r12s16.sjc.aristanetworks.com/172.24.32.16:9104-longCompactions-1439917700325" #161 daemon prio=5 os_prio=0 tid=0x00007fbf54041800 nid=0xcb waiting on condition [0x00007fbf38208000]
  178.    java.lang.Thread.State: WAITING (parking)
  179.         at sun.misc.Unsafe.park(Native Method)
  180.         - parking to wait for  <0x00000005484030d8> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject)
  181.         at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)
  182.         at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039)
  183.         at java.util.concurrent.PriorityBlockingQueue.take(PriorityBlockingQueue.java:549)
  184.         at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1067)
  185.         at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127)
  186.         at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
  187.         at java.lang.Thread.run(Thread.java:745)
  188.  
  189.    Locked ownable synchronizers:
  190.         - None
  191.  
  192. "regionserver/r12s16.sjc.aristanetworks.com/172.24.32.16:9104-shortCompactions-1439917700316" #160 daemon prio=5 os_prio=0 tid=0x0000000001ba6000 nid=0xca in Object.wait() [0x00007fbf38609000]
  193.    java.lang.Thread.State: TIMED_WAITING (on object monitor)
  194.         at java.lang.Object.wait(Native Method)
  195.         at org.apache.hadoop.hbase.regionserver.wal.SyncFuture.get(SyncFuture.java:167)
  196.         - locked <0x0000000548d43ea8> (a org.apache.hadoop.hbase.regionserver.wal.SyncFuture)
  197.         at org.apache.hadoop.hbase.regionserver.wal.FSHLog.blockOnSync(FSHLog.java:1472)
  198.         at org.apache.hadoop.hbase.regionserver.wal.FSHLog.publishSyncThenBlockOnCompletion(FSHLog.java:1466)
  199.         at org.apache.hadoop.hbase.regionserver.wal.FSHLog.sync(FSHLog.java:1585)
  200.         at org.apache.hadoop.hbase.regionserver.wal.WALUtil.writeCompactionMarker(WALUtil.java:64)
  201.         at org.apache.hadoop.hbase.regionserver.HStore.writeCompactionWalRecord(HStore.java:1289)
  202.         at org.apache.hadoop.hbase.regionserver.HStore.compact(HStore.java:1228)
  203.         at org.apache.hadoop.hbase.regionserver.HRegion.compact(HRegion.java:1798)
  204.         at org.apache.hadoop.hbase.regionserver.CompactSplitThread$CompactionRunner.run(CompactSplitThread.java:519)
  205.         at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142)
  206.         at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
  207.         at java.lang.Thread.run(Thread.java:745)
  208.  
  209.    Locked ownable synchronizers:
  210.         - <0x00000005489c8c80> (a java.util.concurrent.ThreadPoolExecutor$Worker)
  211.  
  212. "ShortCircuitCache_Cleaner" #157 daemon prio=5 os_prio=0 tid=0x00007fbf54262800 nid=0xc7 waiting on condition [0x00007fbf38509000]
  213.    java.lang.Thread.State: TIMED_WAITING (parking)
  214.         at sun.misc.Unsafe.park(Native Method)
  215.         - parking to wait for  <0x000000054860e2d0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject)
  216.         at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
  217.         at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)
  218.         at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1093)
  219.         at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809)
  220.         at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1067)
  221.         at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127)
  222.         at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
  223.         at java.lang.Thread.run(Thread.java:745)
  224.  
  225.    Locked ownable synchronizers:
  226.         - None
  227.  
  228. "RS_OPEN_REGION-r12s16:9104-0" #154 prio=5 os_prio=0 tid=0x00007fbf54472800 nid=0xc4 waiting on condition [0x00007fbf3938b000]
  229.    java.lang.Thread.State: WAITING (parking)
  230.         at sun.misc.Unsafe.park(Native Method)
  231.         - parking to wait for  <0x00000005489d88d0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject)
  232.         at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)
  233.         at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039)
  234.         at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442)
  235.         at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1067)
  236.         at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127)
  237.         at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
  238.         at java.lang.Thread.run(Thread.java:745)
  239.  
  240.    Locked ownable synchronizers:
  241.         - None
  242.  
  243. "org.apache.hadoop.hdfs.PeerCache@500f6b9c" #153 daemon prio=5 os_prio=0 tid=0x00007fbf54463800 nid=0xc3 waiting on condition [0x00007fbf3948c000]
  244.    java.lang.Thread.State: TIMED_WAITING (sleeping)
  245.         at java.lang.Thread.sleep(Native Method)
  246.         at org.apache.hadoop.hdfs.PeerCache.run(PeerCache.java:244)
  247.         at org.apache.hadoop.hdfs.PeerCache.access$000(PeerCache.java:41)
  248.         at org.apache.hadoop.hdfs.PeerCache$1.run(PeerCache.java:119)
  249.         at java.lang.Thread.run(Thread.java:745)
  250.  
  251.    Locked ownable synchronizers:
  252.         - None
  253.  
  254. "SplitLogWorker-r12s16:9104" #151 prio=5 os_prio=0 tid=0x00007fbf5cfdc800 nid=0xc1 in Object.wait() [0x00007fbf38c12000]
  255.    java.lang.Thread.State: TIMED_WAITING (on object monitor)
  256.         at java.lang.Object.wait(Native Method)
  257.         at org.apache.hadoop.hbase.coordination.ZkSplitLogWorkerCoordination.taskLoop(ZkSplitLogWorkerCoordination.java:445)
  258.         - locked <0x00000005486a9f40> (a java.lang.Object)
  259.         at org.apache.hadoop.hbase.regionserver.SplitLogWorker.run(SplitLogWorker.java:142)
  260.         at java.lang.Thread.run(Thread.java:745)
  261.  
  262.    Locked ownable synchronizers:
  263.         - None
  264.  
  265. "r12s16:9104Replication Statistics #0" #150 daemon prio=5 os_prio=0 tid=0x00007fbf5cc44000 nid=0xc0 waiting on condition [0x00007fbf38d13000]
  266.    java.lang.Thread.State: TIMED_WAITING (parking)
  267.         at sun.misc.Unsafe.park(Native Method)
  268.         - parking to wait for  <0x00000005483f0810> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject)
  269.         at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
  270.         at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)
  271.         at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1093)
  272.         at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809)
  273.         at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1067)
  274.         at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127)
  275.         at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
  276.         at java.lang.Thread.run(Thread.java:745)
  277.  
  278.    Locked ownable synchronizers:
  279.         - None
  280.  
  281. "regionserver/r12s16.sjc.aristanetworks.com/172.24.32.16:9104-EventThread" #148 daemon prio=5 os_prio=0 tid=0x00007fbf5c680000 nid=0xbf waiting on condition [0x00007fbf38e14000]
  282.    java.lang.Thread.State: WAITING (parking)
  283.         at sun.misc.Unsafe.park(Native Method)
  284.         - parking to wait for  <0x0000000548a14310> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject)
  285.         at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)
  286.         at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039)
  287.         at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442)
  288.         at org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:494)
  289.  
  290.    Locked ownable synchronizers:
  291.         - None
  292.  
  293. "regionserver/r12s16.sjc.aristanetworks.com/172.24.32.16:9104-SendThread(r12s1.sjc.aristanetworks.com:2181)" #147 daemon prio=5 os_prio=0 tid=0x00007fbf5c8e4000 nid=0xbe runnable [0x00007fbf38f15000]
  294.    java.lang.Thread.State: RUNNABLE
  295.         at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method)
  296.         at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269)
  297.         at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:79)
  298.         at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86)
  299.         - locked <0x00000005489e2ee8> (a sun.nio.ch.Util$2)
  300.         - locked <0x00000005489e2ed0> (a java.util.Collections$UnmodifiableSet)
  301.         - locked <0x0000000548992930> (a sun.nio.ch.EPollSelectorImpl)
  302.         at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97)
  303.         at org.apache.zookeeper.ClientCnxnSocketNIO.doTransport(ClientCnxnSocketNIO.java:349)
  304.         at org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1081)
  305.  
  306.    Locked ownable synchronizers:
  307.         - None
  308.  
  309. "regionserver/r12s16.sjc.aristanetworks.com/172.24.32.16:9104.leaseChecker" #124 daemon prio=5 os_prio=0 tid=0x00007fbf5c8ba000 nid=0xbd sleeping[0x00007fbf39016000]
  310.    java.lang.Thread.State: TIMED_WAITING (sleeping)
  311.         at java.lang.Thread.sleep(Native Method)
  312.         at org.apache.hadoop.hbase.regionserver.Leases.run(Leases.java:94)
  313.         at java.lang.Thread.run(Thread.java:745)
  314.  
  315.    Locked ownable synchronizers:
  316.         - None
  317.  
  318. "r12s16.sjc.aristanetworks.com,9104,1439917659071_ChoreService_1" #146 daemon prio=5 os_prio=0 tid=0x00007fbf5cabd000 nid=0xbc waiting on condition [0x00007fbf39117000]
  319.    java.lang.Thread.State: TIMED_WAITING (parking)
  320.         at sun.misc.Unsafe.park(Native Method)
  321.         - parking to wait for  <0x00000005483f0840> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject)
  322.         at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
  323.         at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)
  324.         at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1093)
  325.         at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809)
  326.         at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1067)
  327.         at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127)
  328.         at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
  329.         at java.lang.Thread.run(Thread.java:745)
  330.  
  331.    Locked ownable synchronizers:
  332.         - None
  333.  
  334. "MemStoreFlusher.1" #144 prio=5 os_prio=0 tid=0x00007fbf5cabb800 nid=0xbb waiting on condition [0x00007fbf39217000]
  335.    java.lang.Thread.State: WAITING (parking)
  336.         at sun.misc.Unsafe.park(Native Method)
  337.         - parking to wait for  <0x000000054bcc63f0> (a java.util.concurrent.CountDownLatch$Sync)
  338.         at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)
  339.         at java.util.concurrent.locks.AbstractQueuedSynchronizer.parkAndCheckInterrupt(AbstractQueuedSynchronizer.java:836)
  340.         at java.util.concurrent.locks.AbstractQueuedSynchronizer.doAcquireSharedInterruptibly(AbstractQueuedSynchronizer.java:997)
  341.         at java.util.concurrent.locks.AbstractQueuedSynchronizer.acquireSharedInterruptibly(AbstractQueuedSynchronizer.java:1304)
  342.         at java.util.concurrent.CountDownLatch.await(CountDownLatch.java:231)
  343.         at org.apache.hadoop.hbase.wal.WALKey.getSequenceId(WALKey.java:305)
  344.         at org.apache.hadoop.hbase.regionserver.HRegion.getNextSequenceId(HRegion.java:2413)
  345.         at org.apache.hadoop.hbase.regionserver.HRegion.internalPrepareFlushCache(HRegion.java:2164)
  346.         at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2046)
  347.         at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2011)
  348.         at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:1903)
  349.         at org.apache.hadoop.hbase.regionserver.HRegion.flush(HRegion.java:1829)
  350.         at org.apache.hadoop.hbase.regionserver.MemStoreFlusher.flushRegion(MemStoreFlusher.java:510)
  351.         at org.apache.hadoop.hbase.regionserver.MemStoreFlusher.flushRegion(MemStoreFlusher.java:471)
  352.         at org.apache.hadoop.hbase.regionserver.MemStoreFlusher.access$800(MemStoreFlusher.java:75)
  353.         at org.apache.hadoop.hbase.regionserver.MemStoreFlusher$FlushHandler.run(MemStoreFlusher.java:259)
  354.         at java.lang.Thread.run(Thread.java:745)
  355.  
  356.    Locked ownable synchronizers:
  357.         - <0x000000055b6e3b98> (a java.util.concurrent.locks.ReentrantReadWriteLock$NonfairSync)
  358.  
  359. "MemStoreFlusher.0" #142 prio=5 os_prio=0 tid=0x00007fbf5cab9800 nid=0xba waiting on condition [0x00007fbf39b01000]
  360.    java.lang.Thread.State: WAITING (parking)
  361.         at sun.misc.Unsafe.park(Native Method)
  362.         - parking to wait for  <0x000000054ba6c0c0> (a java.util.concurrent.CountDownLatch$Sync)
  363.         at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)
  364.         at java.util.concurrent.locks.AbstractQueuedSynchronizer.parkAndCheckInterrupt(AbstractQueuedSynchronizer.java:836)
  365.         at java.util.concurrent.locks.AbstractQueuedSynchronizer.doAcquireSharedInterruptibly(AbstractQueuedSynchronizer.java:997)
  366.         at java.util.concurrent.locks.AbstractQueuedSynchronizer.acquireSharedInterruptibly(AbstractQueuedSynchronizer.java:1304)
  367.         at java.util.concurrent.CountDownLatch.await(CountDownLatch.java:231)
  368.         at org.apache.hadoop.hbase.wal.WALKey.getSequenceId(WALKey.java:305)
  369.         at org.apache.hadoop.hbase.regionserver.HRegion.getNextSequenceId(HRegion.java:2413)
  370.         at org.apache.hadoop.hbase.regionserver.HRegion.internalPrepareFlushCache(HRegion.java:2164)
  371.         at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2046)
  372.         at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2011)
  373.         at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:1903)
  374.         at org.apache.hadoop.hbase.regionserver.HRegion.flush(HRegion.java:1829)
  375.         at org.apache.hadoop.hbase.regionserver.MemStoreFlusher.flushRegion(MemStoreFlusher.java:510)
  376.         at org.apache.hadoop.hbase.regionserver.MemStoreFlusher.flushRegion(MemStoreFlusher.java:471)
  377.         at org.apache.hadoop.hbase.regionserver.MemStoreFlusher.access$800(MemStoreFlusher.java:75)
  378.         at org.apache.hadoop.hbase.regionserver.MemStoreFlusher$FlushHandler.run(MemStoreFlusher.java:259)
  379.         at java.lang.Thread.run(Thread.java:745)
  380.  
  381.    Locked ownable synchronizers:
  382.         - <0x000000054945c538> (a java.util.concurrent.locks.ReentrantReadWriteLock$NonfairSync)
  383.  
  384. "regionserver/r12s16.sjc.aristanetworks.com/172.24.32.16:9104.logRoller" #120 daemon prio=5 os_prio=0 tid=0x00007fbf5cd4a800 nid=0xb9 runnable [0x00007fbf39c03000]
  385.    java.lang.Thread.State: RUNNABLE
  386.         at java.util.concurrent.locks.AbstractQueuedSynchronizer.tryAcquireSharedNanos(AbstractQueuedSynchronizer.java:1328)
  387.         at java.util.concurrent.CountDownLatch.await(CountDownLatch.java:277)
  388.         at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SafePointZigZagLatch.waitSafePoint(FSHLog.java:1851)
  389.         at org.apache.hadoop.hbase.regionserver.wal.FSHLog.replaceWriter(FSHLog.java:938)
  390.         at org.apache.hadoop.hbase.regionserver.wal.FSHLog.rollWriter(FSHLog.java:721)
  391.         at org.apache.hadoop.hbase.regionserver.LogRoller.run(LogRoller.java:137)
  392.         at java.lang.Thread.run(Thread.java:745)
  393.  
  394.    Locked ownable synchronizers:
  395.         - <0x000000054840b910> (a java.util.concurrent.locks.ReentrantLock$NonfairSync)
  396.         - <0x000000054874e318> (a java.util.concurrent.locks.ReentrantLock$FairSync)
  397.  
  398. "HBase-Metrics2-1" #141 daemon prio=5 os_prio=0 tid=0x00007fbf5c68d000 nid=0xb8 runnable [0x00007fbf3958d000]
  399.    java.lang.Thread.State: TIMED_WAITING (parking)
  400.         at sun.misc.Unsafe.park(Native Method)
  401.         - parking to wait for  <0x00000005483d34d8> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject)
  402.         at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
  403.         at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)
  404.         at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1093)
  405.         at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809)
  406.         at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1067)
  407.         at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127)
  408.         at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
  409.         at java.lang.Thread.run(Thread.java:745)
  410.  
  411.    Locked ownable synchronizers:
  412.         - None
  413.  
  414. "sync.4" #139 prio=5 os_prio=0 tid=0x00007fbf5cbf5000 nid=0xb7 waiting on condition [0x00007fbf3968e000]
  415.    java.lang.Thread.State: WAITING (parking)
  416.         at sun.misc.Unsafe.park(Native Method)
  417.         - parking to wait for  <0x00000005483d3508> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject)
  418.         at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)
  419.         at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039)
  420.         at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442)
  421.         at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:1323)
  422.         at java.lang.Thread.run(Thread.java:745)
  423.  
  424.    Locked ownable synchronizers:
  425.         - None
  426.  
  427. "sync.3" #138 prio=5 os_prio=0 tid=0x00007fbf5cbf3800 nid=0xb6 waiting on condition [0x00007fbf3978f000]
  428.    java.lang.Thread.State: WAITING (parking)
  429.         at sun.misc.Unsafe.park(Native Method)
  430.         - parking to wait for  <0x00000005483d34c0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject)
  431.         at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)
  432.         at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039)
  433.         at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442)
  434.         at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:1323)
  435.         at java.lang.Thread.run(Thread.java:745)
  436.  
  437.    Locked ownable synchronizers:
  438.         - None
  439.  
  440. "sync.2" #137 prio=5 os_prio=0 tid=0x00007fbf5c937000 nid=0xb5 waiting on condition [0x00007fbf39890000]
  441.    java.lang.Thread.State: WAITING (parking)
  442.         at sun.misc.Unsafe.park(Native Method)
  443.         - parking to wait for  <0x00000005483d3478> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject)
  444.         at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)
  445.         at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039)
  446.         at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442)
  447.         at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:1323)
  448.         at java.lang.Thread.run(Thread.java:745)
  449.  
  450.    Locked ownable synchronizers:
  451.         - None
  452.  
  453. "sync.1" #136 prio=5 os_prio=0 tid=0x00007fbf5c6c0000 nid=0xb4 waiting on condition [0x00007fbf39991000]
  454.    java.lang.Thread.State: WAITING (parking)
  455.         at sun.misc.Unsafe.park(Native Method)
  456.         - parking to wait for  <0x00000005483d3430> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject)
  457.         at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)
  458.         at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039)
  459.         at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442)
  460.         at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:1323)
  461.         at java.lang.Thread.run(Thread.java:745)
  462.  
  463.    Locked ownable synchronizers:
  464.         - None
  465.  
  466. "sync.0" #135 prio=5 os_prio=0 tid=0x00007fbf5c6be800 nid=0xb3 waiting on condition [0x00007fbf3a014000]
  467.    java.lang.Thread.State: WAITING (parking)
  468.         at sun.misc.Unsafe.park(Native Method)
  469.         - parking to wait for  <0x00000005483d33e8> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject)
  470.         at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)
  471.         at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039)
  472.         at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442)
  473.         at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:1323)
  474.         at java.lang.Thread.run(Thread.java:745)
  475.  
  476.    Locked ownable synchronizers:
  477.         - None
  478.  
  479. "regionserver/r12s16.sjc.aristanetworks.com/172.24.32.16:9104.append-pool1-t1" #140 prio=5 os_prio=0 tid=0x00007fbf5cc61800 nid=0xb2 in Object.wait() [0x00007fbf3a115000]
  480.    java.lang.Thread.State: TIMED_WAITING (on object monitor)
  481.         at java.lang.Object.wait(Native Method)
  482.         at java.lang.Object.wait(Object.java:460)
  483.         at org.apache.hadoop.hbase.regionserver.wal.FSHLog$RingBufferEventHandler.attainSafePoint(FSHLog.java:2024)
  484.         - locked <0x0000000548756b60> (a java.lang.Object)
  485.         at org.apache.hadoop.hbase.regionserver.wal.FSHLog$RingBufferEventHandler.onEvent(FSHLog.java:1999)
  486.         at org.apache.hadoop.hbase.regionserver.wal.FSHLog$RingBufferEventHandler.onEvent(FSHLog.java:1910)
  487.         at com.lmax.disruptor.BatchEventProcessor.run(BatchEventProcessor.java:128)
  488.         at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142)
  489.         at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
  490.         at java.lang.Thread.run(Thread.java:745)
  491.  
  492.    Locked ownable synchronizers:
  493.         - <0x0000000548293df0> (a java.util.concurrent.ThreadPoolExecutor$Worker)
  494.  
  495. "LeaseRenewer:aeris@0.namenode.sjc.aristanetworks.com:8820" #133 daemon prio=5 os_prio=0 tid=0x00007fbf5c9d2800 nid=0xb0 waiting on condition [0x00007fbf3a216000]
  496.    java.lang.Thread.State: TIMED_WAITING (sleeping)
  497.         at java.lang.Thread.sleep(Native Method)
  498.         at org.apache.hadoop.hdfs.LeaseRenewer.run(LeaseRenewer.java:438)
  499.         at org.apache.hadoop.hdfs.LeaseRenewer.access$700(LeaseRenewer.java:71)
  500.         at org.apache.hadoop.hdfs.LeaseRenewer$1.run(LeaseRenewer.java:298)
  501.         at java.lang.Thread.run(Thread.java:745)
  502.  
  503.    Locked ownable synchronizers:
  504.         - None
  505.  
  506. "IPC Parameter Sending Thread #0" #131 daemon prio=5 os_prio=0 tid=0x00007fbf5c77e800 nid=0xae waiting on condition [0x00007fbf3a418000]
  507.    java.lang.Thread.State: TIMED_WAITING (parking)
  508.         at sun.misc.Unsafe.park(Native Method)
  509.         - parking to wait for  <0x00000005484a49a0> (a java.util.concurrent.SynchronousQueue$TransferStack)
  510.         at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
  511.         at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460)
  512.         at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362)
  513.         at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941)
  514.         at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1066)
  515.         at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127)
  516.         at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
  517.         at java.lang.Thread.run(Thread.java:745)
  518.  
  519.    Locked ownable synchronizers:
  520.         - None
  521.  
  522. "LruBlockCacheStatsExecutor" #129 daemon prio=5 os_prio=0 tid=0x00007fbf5ca34000 nid=0xac waiting on condition [0x00007fbf3a61a000]
  523.    java.lang.Thread.State: TIMED_WAITING (parking)
  524.         at sun.misc.Unsafe.park(Native Method)
  525.         - parking to wait for  <0x00000005483f08b8> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject)
  526.         at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
  527.         at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)
  528.         at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1093)
  529.         at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809)
  530.         at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1067)
  531.         at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127)
  532.         at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
  533.         at java.lang.Thread.run(Thread.java:745)
  534.  
  535.    Locked ownable synchronizers:
  536.         - None
  537.  
  538. "regionserver/r12s16.sjc.aristanetworks.com/172.24.32.16:9104.LruBlockCache.EvictionThread" #127 daemon prio=5 os_prio=0 tid=0x00007fbf5ca4d800 nid=0xab in Object.wait() [0x00007fbf3aaa7000]
  539.    java.lang.Thread.State: TIMED_WAITING (on object monitor)
  540.         at java.lang.Object.wait(Native Method)
  541.         at org.apache.hadoop.hbase.io.hfile.LruBlockCache$EvictionThread.run(LruBlockCache.java:803)
  542.         - locked <0x00000005483f08e8> (a org.apache.hadoop.hbase.io.hfile.LruBlockCache$EvictionThread)
  543.         at java.lang.Thread.run(Thread.java:745)
  544.  
  545.    Locked ownable synchronizers:
  546.         - None
  547.  
  548. "IPC Client (1261031890) connection to r12s3.sjc.aristanetworks.com/172.24.32.3:9102 from aeris" #126 daemon prio=5 os_prio=0 tid=0x00007fbf5c429800 nid=0xaa in Object.wait() [0x00007fbf3a91b000]
  549.    java.lang.Thread.State: TIMED_WAITING (on object monitor)
  550.         at java.lang.Object.wait(Native Method)
  551.         at org.apache.hadoop.hbase.ipc.RpcClientImpl$Connection.waitForWork(RpcClientImpl.java:542)
  552.         - locked <0x0000000548464f58> (a org.apache.hadoop.hbase.ipc.RpcClientImpl$Connection)
  553.         at org.apache.hadoop.hbase.ipc.RpcClientImpl$Connection.run(RpcClientImpl.java:557)
  554.  
  555.    Locked ownable synchronizers:
  556.         - None
  557.  
  558. "JvmPauseMonitor" #123 daemon prio=5 os_prio=0 tid=0x00007fbf5c44b800 nid=0xa9 waiting on condition [0x00007fbf3aba8000]
  559.    java.lang.Thread.State: TIMED_WAITING (sleeping)
  560.         at java.lang.Thread.sleep(Native Method)
  561.         at org.apache.hadoop.hbase.util.JvmPauseMonitor$Monitor.run(JvmPauseMonitor.java:159)
  562.         at java.lang.Thread.run(Thread.java:745)
  563.  
  564.    Locked ownable synchronizers:
  565.         - None
  566.  
  567. "regionserver/r12s16.sjc.aristanetworks.com/172.24.32.16:9104-EventThread" #122 daemon prio=5 os_prio=0 tid=0x00007fbf5c3ed000 nid=0xa8 waiting on condition [0x00007fbf3aea9000]
  568.    java.lang.Thread.State: WAITING (parking)
  569.         at sun.misc.Unsafe.park(Native Method)
  570.         - parking to wait for  <0x00000005483f0900> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject)
  571.         at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)
  572.         at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039)
  573.         at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442)
  574.         at org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:494)
  575.  
  576.    Locked ownable synchronizers:
  577.         - None
  578.  
  579. "regionserver/r12s16.sjc.aristanetworks.com/172.24.32.16:9104-SendThread(r12s9.sjc.aristanetworks.com:2181)" #121 daemon prio=5 os_prio=0 tid=0x00007fbf5c3ec800 nid=0xa7 runnable [0x00007fbf3afaa000]
  580.    java.lang.Thread.State: RUNNABLE
  581.         at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method)
  582.         at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269)
  583.         at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:79)
  584.         at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86)
  585.         - locked <0x00000005483d3250> (a sun.nio.ch.Util$2)
  586.         - locked <0x00000005483d32c8> (a java.util.Collections$UnmodifiableSet)
  587.         - locked <0x00000005480581d0> (a sun.nio.ch.EPollSelectorImpl)
  588.         at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97)
  589.         at org.apache.zookeeper.ClientCnxnSocketNIO.doTransport(ClientCnxnSocketNIO.java:349)
  590.         at org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1081)
  591.  
  592.    Locked ownable synchronizers:
  593.         - None
  594.  
  595. "regionserver/r12s16.sjc.aristanetworks.com/172.24.32.16:9104" #25 prio=5 os_prio=0 tid=0x00007fbf71716000 nid=0xa6 in Object.wait() [0x00007fbf3b0ab000]
  596.    java.lang.Thread.State: TIMED_WAITING (on object monitor)
  597.         at java.lang.Object.wait(Native Method)
  598.         at org.apache.hadoop.hbase.util.Sleeper.sleep(Sleeper.java:92)
  599.         - locked <0x00000005483aa0f8> (a java.lang.Object)
  600.         at org.apache.hadoop.hbase.util.Sleeper.sleep(Sleeper.java:56)
  601.         at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:951)
  602.         at java.lang.Thread.run(Thread.java:745)
  603.  
  604.    Locked ownable synchronizers:
  605.         - None
  606.  
  607. "Timer-0" #119 daemon prio=5 os_prio=0 tid=0x00007fbf716f3000 nid=0xa5 in Object.wait() [0x00007fbf3b1ac000]
  608.    java.lang.Thread.State: TIMED_WAITING (on object monitor)
  609.         at java.lang.Object.wait(Native Method)
  610.         at java.util.TimerThread.mainLoop(Timer.java:552)
  611.         - locked <0x00000005483d3268> (a java.util.TaskQueue)
  612.         at java.util.TimerThread.run(Timer.java:505)
  613.  
  614.    Locked ownable synchronizers:
  615.         - None
  616.  
  617. "1073741635@qtp-864138492-1 - Acceptor0 SelectChannelConnector@0.0.0.0:9103" #118 daemon prio=5 os_prio=0 tid=0x00007fbf71646000 nid=0xa4 runnable [0x00007fbf3b2ad000]
  618.    java.lang.Thread.State: RUNNABLE
  619.         at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method)
  620.         at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269)
  621.         at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:79)
  622.         at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86)
  623.         - locked <0x00000005483d3220> (a sun.nio.ch.Util$2)
  624.         - locked <0x00000005483d3208> (a java.util.Collections$UnmodifiableSet)
  625.         - locked <0x0000000548058140> (a sun.nio.ch.EPollSelectorImpl)
  626.         at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97)
  627.         at org.mortbay.io.nio.SelectorManager$SelectSet.doSelect(SelectorManager.java:498)
  628.         at org.mortbay.io.nio.SelectorManager.doSelect(SelectorManager.java:192)
  629.         at org.mortbay.jetty.nio.SelectChannelConnector.accept(SelectChannelConnector.java:124)
  630.         at org.mortbay.jetty.AbstractConnector$Acceptor.run(AbstractConnector.java:708)
  631.         at org.mortbay.thread.QueuedThreadPool$PoolThread.run(QueuedThreadPool.java:582)
  632.  
  633.    Locked ownable synchronizers:
  634.         - None
  635.  
  636. "ReplicationRpcServer.handler=2,queue=0,port=9104" #115 daemon prio=5 os_prio=0 tid=0x00007fbf71558800 nid=0xa2 waiting on condition [0x00007fbf3b6af000]
  637.    java.lang.Thread.State: WAITING (parking)
  638.         at sun.misc.Unsafe.park(Native Method)
  639.         - parking to wait for  <0x00000005480250f0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject)
  640.         at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)
  641.         at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039)
  642.         at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442)
  643.         at org.apache.hadoop.hbase.ipc.RpcExecutor.consumerLoop(RpcExecutor.java:127)
  644.         at org.apache.hadoop.hbase.ipc.RpcExecutor$1.run(RpcExecutor.java:107)
  645.         at java.lang.Thread.run(Thread.java:745)
  646.  
  647.    Locked ownable synchronizers:
  648.         - None
  649.  
  650. "ReplicationRpcServer.handler=1,queue=0,port=9104" #114 daemon prio=5 os_prio=0 tid=0x00007fbf71556800 nid=0xa1 waiting on condition [0x00007fbf3b7b0000]
  651.    java.lang.Thread.State: WAITING (parking)
  652.         at sun.misc.Unsafe.park(Native Method)
  653.         - parking to wait for  <0x00000005480250f0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject)
  654.         at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)
  655.         at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039)
  656.         at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442)
  657.         at org.apache.hadoop.hbase.ipc.RpcExecutor.consumerLoop(RpcExecutor.java:127)
  658.         at org.apache.hadoop.hbase.ipc.RpcExecutor$1.run(RpcExecutor.java:107)
  659.         at java.lang.Thread.run(Thread.java:745)
  660.  
  661.    Locked ownable synchronizers:
  662.         - None
  663.  
  664. "ReplicationRpcServer.handler=0,queue=0,port=9104" #113 daemon prio=5 os_prio=0 tid=0x00007fbf71554800 nid=0xa0 waiting on condition [0x00007fbf3b8b1000]
  665.    java.lang.Thread.State: WAITING (parking)
  666.         at sun.misc.Unsafe.park(Native Method)
  667.         - parking to wait for  <0x00000005480250f0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject)
  668.         at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)
  669.         at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039)
  670.         at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442)
  671.         at org.apache.hadoop.hbase.ipc.RpcExecutor.consumerLoop(RpcExecutor.java:127)
  672.         at org.apache.hadoop.hbase.ipc.RpcExecutor$1.run(RpcExecutor.java:107)
  673.         at java.lang.Thread.run(Thread.java:745)
  674.  
  675.    Locked ownable synchronizers:
  676.         - None
  677.  
  678. "PriorityRpcServer.handler=19,queue=1,port=9104" #112 daemon prio=5 os_prio=0 tid=0x00007fbf71552800 nid=0x9f waiting on condition [0x00007fbf3b9b2000]
  679.    java.lang.Thread.State: WAITING (parking)
  680.         at sun.misc.Unsafe.park(Native Method)
  681.         - parking to wait for  <0x00000005480253f0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject)
  682.         at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)
  683.         at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039)
  684.         at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442)
  685.         at org.apache.hadoop.hbase.ipc.RpcExecutor.consumerLoop(RpcExecutor.java:127)
  686.         at org.apache.hadoop.hbase.ipc.RpcExecutor$1.run(RpcExecutor.java:107)
  687.         at java.lang.Thread.run(Thread.java:745)
  688.  
  689.    Locked ownable synchronizers:
  690.         - None
  691.  
  692. "PriorityRpcServer.handler=18,queue=0,port=9104" #111 daemon prio=5 os_prio=0 tid=0x00007fbf71550800 nid=0x9e waiting on condition [0x00007fbf3bab3000]
  693.    java.lang.Thread.State: WAITING (parking)
  694.         at sun.misc.Unsafe.park(Native Method)
  695.         - parking to wait for  <0x00000005480265a0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject)
  696.         at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)
  697.         at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039)
  698.         at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442)
  699.         at org.apache.hadoop.hbase.ipc.RpcExecutor.consumerLoop(RpcExecutor.java:127)
  700.         at org.apache.hadoop.hbase.ipc.RpcExecutor$1.run(RpcExecutor.java:107)
  701.         at java.lang.Thread.run(Thread.java:745)
  702.  
  703.    Locked ownable synchronizers:
  704.         - None
  705.  
  706. "PriorityRpcServer.handler=17,queue=1,port=9104" #110 daemon prio=5 os_prio=0 tid=0x00007fbf7154e800 nid=0x9d waiting on condition [0x00007fbf3bbb4000]
  707.    java.lang.Thread.State: WAITING (parking)
  708.         at sun.misc.Unsafe.park(Native Method)
  709.         - parking to wait for  <0x00000005480253f0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject)
  710.         at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)
  711.         at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039)
  712.         at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442)
  713.         at org.apache.hadoop.hbase.ipc.RpcExecutor.consumerLoop(RpcExecutor.java:127)
  714.         at org.apache.hadoop.hbase.ipc.RpcExecutor$1.run(RpcExecutor.java:107)
  715.         at java.lang.Thread.run(Thread.java:745)
  716.  
  717.    Locked ownable synchronizers:
  718.         - None
  719.  
  720. "PriorityRpcServer.handler=16,queue=0,port=9104" #109 daemon prio=5 os_prio=0 tid=0x00007fbf7154d000 nid=0x9c waiting on condition [0x00007fbf3bcb5000]
  721.    java.lang.Thread.State: WAITING (parking)
  722.         at sun.misc.Unsafe.park(Native Method)
  723.         - parking to wait for  <0x00000005480265a0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject)
  724.         at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)
  725.         at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039)
  726.         at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442)
  727.         at org.apache.hadoop.hbase.ipc.RpcExecutor.consumerLoop(RpcExecutor.java:127)
  728.         at org.apache.hadoop.hbase.ipc.RpcExecutor$1.run(RpcExecutor.java:107)
  729.         at java.lang.Thread.run(Thread.java:745)
  730.  
  731.    Locked ownable synchronizers:
  732.         - None
  733.  
  734. "PriorityRpcServer.handler=15,queue=1,port=9104" #108 daemon prio=5 os_prio=0 tid=0x00007fbf715d4800 nid=0x9b waiting on condition [0x00007fbf3bdb6000]
  735.    java.lang.Thread.State: WAITING (parking)
  736.         at sun.misc.Unsafe.park(Native Method)
  737.         - parking to wait for  <0x00000005480253f0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject)
  738.         at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)
  739.         at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039)
  740.         at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442)
  741.         at org.apache.hadoop.hbase.ipc.RpcExecutor.consumerLoop(RpcExecutor.java:127)
  742.         at org.apache.hadoop.hbase.ipc.RpcExecutor$1.run(RpcExecutor.java:107)
  743.         at java.lang.Thread.run(Thread.java:745)
  744.  
  745.    Locked ownable synchronizers:
  746.         - None
  747.  
  748. "PriorityRpcServer.handler=14,queue=0,port=9104" #107 daemon prio=5 os_prio=0 tid=0x00007fbf715d2800 nid=0x9a waiting on condition [0x00007fbf3beb7000]
  749.    java.lang.Thread.State: WAITING (parking)
  750.         at sun.misc.Unsafe.park(Native Method)
  751.         - parking to wait for  <0x00000005480265a0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject)
  752.         at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)
  753.         at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039)
  754.         at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442)
  755.         at org.apache.hadoop.hbase.ipc.RpcExecutor.consumerLoop(RpcExecutor.java:127)
  756.         at org.apache.hadoop.hbase.ipc.RpcExecutor$1.run(RpcExecutor.java:107)
  757.         at java.lang.Thread.run(Thread.java:745)
  758.  
  759.    Locked ownable synchronizers:
  760.         - None
  761.  
  762. "PriorityRpcServer.handler=13,queue=1,port=9104" #106 daemon prio=5 os_prio=0 tid=0x00007fbf715d0800 nid=0x99 waiting on condition [0x00007fbf3bfb8000]
  763.    java.lang.Thread.State: WAITING (parking)
  764.         at sun.misc.Unsafe.park(Native Method)
  765.         - parking to wait for  <0x00000005480253f0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject)
  766.         at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)
  767.         at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039)
  768.         at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442)
  769.         at org.apache.hadoop.hbase.ipc.RpcExecutor.consumerLoop(RpcExecutor.java:127)
  770.         at org.apache.hadoop.hbase.ipc.RpcExecutor$1.run(RpcExecutor.java:107)
  771.         at java.lang.Thread.run(Thread.java:745)
  772.  
  773.    Locked ownable synchronizers:
  774.         - None
  775.  
  776. "PriorityRpcServer.handler=12,queue=0,port=9104" #105 daemon prio=5 os_prio=0 tid=0x00007fbf715cf000 nid=0x98 waiting on condition [0x00007fbf3c0b9000]
  777.    java.lang.Thread.State: WAITING (parking)
  778.         at sun.misc.Unsafe.park(Native Method)
  779.         - parking to wait for  <0x00000005480265a0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject)
  780.         at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)
  781.         at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039)
  782.         at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442)
  783.         at org.apache.hadoop.hbase.ipc.RpcExecutor.consumerLoop(RpcExecutor.java:127)
  784.         at org.apache.hadoop.hbase.ipc.RpcExecutor$1.run(RpcExecutor.java:107)
  785.         at java.lang.Thread.run(Thread.java:745)
  786.  
  787.    Locked ownable synchronizers:
  788.         - None
  789.  
  790. "PriorityRpcServer.handler=11,queue=1,port=9104" #104 daemon prio=5 os_prio=0 tid=0x00007fbf715cc800 nid=0x97 waiting on condition [0x00007fbf3c1ba000]
  791.    java.lang.Thread.State: WAITING (parking)
  792.         at sun.misc.Unsafe.park(Native Method)
  793.         - parking to wait for  <0x00000005480253f0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject)
  794.         at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)
  795.         at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039)
  796.         at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442)
  797.         at org.apache.hadoop.hbase.ipc.RpcExecutor.consumerLoop(RpcExecutor.java:127)
  798.         at org.apache.hadoop.hbase.ipc.RpcExecutor$1.run(RpcExecutor.java:107)
  799.         at java.lang.Thread.run(Thread.java:745)
  800.  
  801.    Locked ownable synchronizers:
  802.         - None
  803.  
  804. "PriorityRpcServer.handler=10,queue=0,port=9104" #103 daemon prio=5 os_prio=0 tid=0x00007fbf715cb000 nid=0x96 waiting on condition [0x00007fbf3c2bb000]
  805.    java.lang.Thread.State: WAITING (parking)
  806.         at sun.misc.Unsafe.park(Native Method)
  807.         - parking to wait for  <0x00000005480265a0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject)
  808.         at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)
  809.         at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039)
  810.         at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442)
  811.         at org.apache.hadoop.hbase.ipc.RpcExecutor.consumerLoop(RpcExecutor.java:127)
  812.         at org.apache.hadoop.hbase.ipc.RpcExecutor$1.run(RpcExecutor.java:107)
  813.         at java.lang.Thread.run(Thread.java:745)
  814.  
  815.    Locked ownable synchronizers:
  816.         - None
  817.  
  818. "PriorityRpcServer.handler=9,queue=1,port=9104" #102 daemon prio=5 os_prio=0 tid=0x00007fbf715c9000 nid=0x95 waiting on condition [0x00007fbf3c3bc000]
  819.    java.lang.Thread.State: WAITING (parking)
  820.         at sun.misc.Unsafe.park(Native Method)
  821.         - parking to wait for  <0x00000005480253f0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject)
  822.         at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)
  823.         at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039)
  824.         at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442)
  825.         at org.apache.hadoop.hbase.ipc.RpcExecutor.consumerLoop(RpcExecutor.java:127)
  826.         at org.apache.hadoop.hbase.ipc.RpcExecutor$1.run(RpcExecutor.java:107)
  827.         at java.lang.Thread.run(Thread.java:745)
  828.  
  829.    Locked ownable synchronizers:
  830.         - None
  831.  
  832. "PriorityRpcServer.handler=8,queue=0,port=9104" #101 daemon prio=5 os_prio=0 tid=0x00007fbf715c7000 nid=0x94 waiting on condition [0x00007fbf3c4bd000]
  833.    java.lang.Thread.State: WAITING (parking)
  834.         at sun.misc.Unsafe.park(Native Method)
  835.         - parking to wait for  <0x00000005480265a0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject)
  836.         at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)
  837.         at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039)
  838.         at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442)
  839.         at org.apache.hadoop.hbase.ipc.RpcExecutor.consumerLoop(RpcExecutor.java:127)
  840.         at org.apache.hadoop.hbase.ipc.RpcExecutor$1.run(RpcExecutor.java:107)
  841.         at java.lang.Thread.run(Thread.java:745)
  842.  
  843.    Locked ownable synchronizers:
  844.         - None
  845.  
  846. "PriorityRpcServer.handler=7,queue=1,port=9104" #100 daemon prio=5 os_prio=0 tid=0x00007fbf715c4800 nid=0x93 waiting on condition [0x00007fbf3c5be000]
  847.    java.lang.Thread.State: WAITING (parking)
  848.         at sun.misc.Unsafe.park(Native Method)
  849.         - parking to wait for  <0x00000005480253f0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject)
  850.         at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)
  851.         at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039)
  852.         at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442)
  853.         at org.apache.hadoop.hbase.ipc.RpcExecutor.consumerLoop(RpcExecutor.java:127)
  854.         at org.apache.hadoop.hbase.ipc.RpcExecutor$1.run(RpcExecutor.java:107)
  855.         at java.lang.Thread.run(Thread.java:745)
  856.  
  857.    Locked ownable synchronizers:
  858.         - None
  859.  
  860. "PriorityRpcServer.handler=6,queue=0,port=9104" #99 daemon prio=5 os_prio=0 tid=0x00007fbf715c3000 nid=0x92 waiting on condition [0x00007fbf3c6bf000]
  861.    java.lang.Thread.State: WAITING (parking)
  862.         at sun.misc.Unsafe.park(Native Method)
  863.         - parking to wait for  <0x00000005480265a0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject)
  864.         at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)
  865.         at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039)
  866.         at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442)
  867.         at org.apache.hadoop.hbase.ipc.RpcExecutor.consumerLoop(RpcExecutor.java:127)
  868.         at org.apache.hadoop.hbase.ipc.RpcExecutor$1.run(RpcExecutor.java:107)
  869.         at java.lang.Thread.run(Thread.java:745)
  870.  
  871.    Locked ownable synchronizers:
  872.         - None
  873.  
  874. "PriorityRpcServer.handler=5,queue=1,port=9104" #98 daemon prio=5 os_prio=0 tid=0x00007fbf715c1000 nid=0x91 waiting on condition [0x00007fbf3c7c0000]
  875.    java.lang.Thread.State: WAITING (parking)
  876.         at sun.misc.Unsafe.park(Native Method)
  877.         - parking to wait for  <0x00000005480253f0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject)
  878.         at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)
  879.         at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039)
  880.         at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442)
  881.         at org.apache.hadoop.hbase.ipc.RpcExecutor.consumerLoop(RpcExecutor.java:127)
  882.         at org.apache.hadoop.hbase.ipc.RpcExecutor$1.run(RpcExecutor.java:107)
  883.         at java.lang.Thread.run(Thread.java:745)
  884.  
  885.    Locked ownable synchronizers:
  886.         - None
  887.  
  888. "PriorityRpcServer.handler=4,queue=0,port=9104" #97 daemon prio=5 os_prio=0 tid=0x00007fbf715be800 nid=0x90 waiting on condition [0x00007fbf3c8c1000]
  889.    java.lang.Thread.State: WAITING (parking)
  890.         at sun.misc.Unsafe.park(Native Method)
  891.         - parking to wait for  <0x00000005480265a0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject)
  892.         at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)
  893.         at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039)
  894.         at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442)
  895.         at org.apache.hadoop.hbase.ipc.RpcExecutor.consumerLoop(RpcExecutor.java:127)
  896.         at org.apache.hadoop.hbase.ipc.RpcExecutor$1.run(RpcExecutor.java:107)
  897.         at java.lang.Thread.run(Thread.java:745)
  898.  
  899.    Locked ownable synchronizers:
  900.         - None
  901.  
  902. "PriorityRpcServer.handler=3,queue=1,port=9104" #96 daemon prio=5 os_prio=0 tid=0x00007fbf715bd000 nid=0x8f waiting on condition [0x00007fbf3c9c2000]
  903.    java.lang.Thread.State: WAITING (parking)
  904.         at sun.misc.Unsafe.park(Native Method)
  905.         - parking to wait for  <0x00000005480253f0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject)
  906.         at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)
  907.         at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039)
  908.         at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442)
  909.         at org.apache.hadoop.hbase.ipc.RpcExecutor.consumerLoop(RpcExecutor.java:127)
  910.         at org.apache.hadoop.hbase.ipc.RpcExecutor$1.run(RpcExecutor.java:107)
  911.         at java.lang.Thread.run(Thread.java:745)
  912.  
  913.    Locked ownable synchronizers:
  914.         - None
  915.  
  916. "PriorityRpcServer.handler=2,queue=0,port=9104" #95 daemon prio=5 os_prio=0 tid=0x00007fbf715bb000 nid=0x8e waiting on condition [0x00007fbf3cac3000]
  917.    java.lang.Thread.State: WAITING (parking)
  918.         at sun.misc.Unsafe.park(Native Method)
  919.         - parking to wait for  <0x00000005480265a0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject)
  920.         at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)
  921.         at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039)
  922.         at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442)
  923.         at org.apache.hadoop.hbase.ipc.RpcExecutor.consumerLoop(RpcExecutor.java:127)
  924.         at org.apache.hadoop.hbase.ipc.RpcExecutor$1.run(RpcExecutor.java:107)
  925.         at java.lang.Thread.run(Thread.java:745)
  926.  
  927.    Locked ownable synchronizers:
  928.         - None
  929.  
  930. "PriorityRpcServer.handler=1,queue=1,port=9104" #94 daemon prio=5 os_prio=0 tid=0x00007fbf715b9000 nid=0x8d waiting on condition [0x00007fbf3cbc4000]
  931.    java.lang.Thread.State: WAITING (parking)
  932.         at sun.misc.Unsafe.park(Native Method)
  933.         - parking to wait for  <0x00000005480253f0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject)
  934.         at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)
  935.         at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039)
  936.         at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442)
  937.         at org.apache.hadoop.hbase.ipc.RpcExecutor.consumerLoop(RpcExecutor.java:127)
  938.         at org.apache.hadoop.hbase.ipc.RpcExecutor$1.run(RpcExecutor.java:107)
  939.         at java.lang.Thread.run(Thread.java:745)
  940.  
  941.    Locked ownable synchronizers:
  942.         - None
  943.  
  944. "PriorityRpcServer.handler=0,queue=0,port=9104" #93 daemon prio=5 os_prio=0 tid=0x00007fbf715b6800 nid=0x8c waiting on condition [0x00007fbf3ccc5000]
  945.    java.lang.Thread.State: WAITING (parking)
  946.         at sun.misc.Unsafe.park(Native Method)
  947.         - parking to wait for  <0x00000005480265a0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject)
  948.         at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)
  949.         at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039)
  950.         at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442)
  951.         at org.apache.hadoop.hbase.ipc.RpcExecutor.consumerLoop(RpcExecutor.java:127)
  952.         at org.apache.hadoop.hbase.ipc.RpcExecutor$1.run(RpcExecutor.java:107)
  953.         at java.lang.Thread.run(Thread.java:745)
  954.  
  955.    Locked ownable synchronizers:
  956.         - None
  957.  
  958. "B.defaultRpcServer.handler=49,queue=4,port=9104" #92 daemon prio=5 os_prio=0 tid=0x00007fbf715b2000 nid=0x8b in Object.wait() [0x00007fbf3cdc6000]
  959.    java.lang.Thread.State: TIMED_WAITING (on object monitor)
  960.         at java.lang.Object.wait(Native Method)
  961.         at org.apache.hadoop.hbase.regionserver.wal.SyncFuture.get(SyncFuture.java:167)
  962.         - locked <0x00000005482ffcd8> (a org.apache.hadoop.hbase.regionserver.wal.SyncFuture)
  963.         at org.apache.hadoop.hbase.regionserver.wal.FSHLog.blockOnSync(FSHLog.java:1472)
  964.         at org.apache.hadoop.hbase.regionserver.wal.FSHLog.publishSyncThenBlockOnCompletion(FSHLog.java:1466)
  965.         at org.apache.hadoop.hbase.regionserver.wal.FSHLog.sync(FSHLog.java:1600)
  966.         at org.apache.hadoop.hbase.regionserver.HRegion.syncOrDefer(HRegion.java:7727)
  967.         at org.apache.hadoop.hbase.regionserver.HRegion.doMiniBatchMutation(HRegion.java:3145)
  968.         at org.apache.hadoop.hbase.regionserver.HRegion.batchMutate(HRegion.java:2793)
  969.         at org.apache.hadoop.hbase.regionserver.HRegion.batchMutate(HRegion.java:2735)
  970.         at org.apache.hadoop.hbase.regionserver.RSRpcServices.doBatchOp(RSRpcServices.java:692)
  971.         at org.apache.hadoop.hbase.regionserver.RSRpcServices.doNonAtomicRegionMutation(RSRpcServices.java:654)
  972.         at org.apache.hadoop.hbase.regionserver.RSRpcServices.multi(RSRpcServices.java:2029)
  973.         at org.apache.hadoop.hbase.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:32213)
  974.         at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:2112)
  975.         at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:101)
  976.         at org.apache.hadoop.hbase.ipc.RpcExecutor.consumerLoop(RpcExecutor.java:130)
  977.         at org.apache.hadoop.hbase.ipc.RpcExecutor$1.run(RpcExecutor.java:107)
  978.         at java.lang.Thread.run(Thread.java:745)
  979.  
  980.    Locked ownable synchronizers:
  981.         - None
  982.  
  983. "B.defaultRpcServer.handler=48,queue=3,port=9104" #91 daemon prio=5 os_prio=0 tid=0x00007fbf715b0000 nid=0x8a in Object.wait() [0x00007fbf3cec7000]
  984.    java.lang.Thread.State: TIMED_WAITING (on object monitor)
  985.         at java.lang.Object.wait(Native Method)
  986.         at org.apache.hadoop.hbase.regionserver.wal.SyncFuture.get(SyncFuture.java:167)
  987.         - locked <0x0000000548b22b68> (a org.apache.hadoop.hbase.regionserver.wal.SyncFuture)
  988.         at org.apache.hadoop.hbase.regionserver.wal.FSHLog.blockOnSync(FSHLog.java:1472)
  989.         at org.apache.hadoop.hbase.regionserver.wal.FSHLog.publishSyncThenBlockOnCompletion(FSHLog.java:1466)
  990.         at org.apache.hadoop.hbase.regionserver.wal.FSHLog.sync(FSHLog.java:1600)
  991.         at org.apache.hadoop.hbase.regionserver.HRegion.syncOrDefer(HRegion.java:7727)
  992.         at org.apache.hadoop.hbase.regionserver.HRegion.doMiniBatchMutation(HRegion.java:3145)
  993.         at org.apache.hadoop.hbase.regionserver.HRegion.batchMutate(HRegion.java:2793)
  994.         at org.apache.hadoop.hbase.regionserver.HRegion.batchMutate(HRegion.java:2735)
  995.         at org.apache.hadoop.hbase.regionserver.RSRpcServices.doBatchOp(RSRpcServices.java:692)
  996.         at org.apache.hadoop.hbase.regionserver.RSRpcServices.doNonAtomicRegionMutation(RSRpcServices.java:654)
  997.         at org.apache.hadoop.hbase.regionserver.RSRpcServices.multi(RSRpcServices.java:2029)
  998.         at org.apache.hadoop.hbase.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:32213)
  999.         at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:2112)
  1000.         at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:101)
  1001.         at org.apache.hadoop.hbase.ipc.RpcExecutor.consumerLoop(RpcExecutor.java:130)
  1002.         at org.apache.hadoop.hbase.ipc.RpcExecutor$1.run(RpcExecutor.java:107)
  1003.         at java.lang.Thread.run(Thread.java:745)
  1004.  
  1005.    Locked ownable synchronizers:
  1006.         - None
  1007.  
  1008. "B.defaultRpcServer.handler=47,queue=2,port=9104" #90 daemon prio=5 os_prio=0 tid=0x00007fbf715ae000 nid=0x89 in Object.wait() [0x00007fbf3cfc8000]
  1009.    java.lang.Thread.State: TIMED_WAITING (on object monitor)
  1010.         at java.lang.Object.wait(Native Method)
  1011.         at org.apache.hadoop.hbase.regionserver.wal.SyncFuture.get(SyncFuture.java:167)
  1012.         - locked <0x0000000548bb0618> (a org.apache.hadoop.hbase.regionserver.wal.SyncFuture)
  1013.         at org.apache.hadoop.hbase.regionserver.wal.FSHLog.blockOnSync(FSHLog.java:1472)
  1014.         at org.apache.hadoop.hbase.regionserver.wal.FSHLog.publishSyncThenBlockOnCompletion(FSHLog.java:1466)
  1015.         at org.apache.hadoop.hbase.regionserver.wal.FSHLog.sync(FSHLog.java:1600)
  1016.         at org.apache.hadoop.hbase.regionserver.HRegion.syncOrDefer(HRegion.java:7727)
  1017.         at org.apache.hadoop.hbase.regionserver.HRegion.doMiniBatchMutation(HRegion.java:3145)
  1018.         at org.apache.hadoop.hbase.regionserver.HRegion.batchMutate(HRegion.java:2793)
  1019.         at org.apache.hadoop.hbase.regionserver.HRegion.batchMutate(HRegion.java:2735)
  1020.         at org.apache.hadoop.hbase.regionserver.RSRpcServices.doBatchOp(RSRpcServices.java:692)
  1021.         at org.apache.hadoop.hbase.regionserver.RSRpcServices.doNonAtomicRegionMutation(RSRpcServices.java:654)
  1022.         at org.apache.hadoop.hbase.regionserver.RSRpcServices.multi(RSRpcServices.java:2029)
  1023.         at org.apache.hadoop.hbase.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:32213)
  1024.         at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:2112)
  1025.         at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:101)
  1026.         at org.apache.hadoop.hbase.ipc.RpcExecutor.consumerLoop(RpcExecutor.java:130)
  1027.         at org.apache.hadoop.hbase.ipc.RpcExecutor$1.run(RpcExecutor.java:107)
  1028.         at java.lang.Thread.run(Thread.java:745)
  1029.  
  1030.    Locked ownable synchronizers:
  1031.         - None
  1032.  
  1033. "B.defaultRpcServer.handler=46,queue=1,port=9104" #89 daemon prio=5 os_prio=0 tid=0x00007fbf715ac000 nid=0x88 in Object.wait() [0x00007fbf3d0c9000]
  1034.    java.lang.Thread.State: TIMED_WAITING (on object monitor)
  1035.         at java.lang.Object.wait(Native Method)
  1036.         at org.apache.hadoop.hbase.regionserver.wal.SyncFuture.get(SyncFuture.java:167)
  1037.         - locked <0x00000005486ea528> (a org.apache.hadoop.hbase.regionserver.wal.SyncFuture)
  1038.         at org.apache.hadoop.hbase.regionserver.wal.FSHLog.blockOnSync(FSHLog.java:1472)
  1039.         at org.apache.hadoop.hbase.regionserver.wal.FSHLog.publishSyncThenBlockOnCompletion(FSHLog.java:1466)
  1040.         at org.apache.hadoop.hbase.regionserver.wal.FSHLog.sync(FSHLog.java:1600)
  1041.         at org.apache.hadoop.hbase.regionserver.HRegion.syncOrDefer(HRegion.java:7727)
  1042.         at org.apache.hadoop.hbase.regionserver.HRegion.doMiniBatchMutation(HRegion.java:3145)
  1043.         at org.apache.hadoop.hbase.regionserver.HRegion.batchMutate(HRegion.java:2793)
  1044.         at org.apache.hadoop.hbase.regionserver.HRegion.batchMutate(HRegion.java:2735)
  1045.         at org.apache.hadoop.hbase.regionserver.RSRpcServices.doBatchOp(RSRpcServices.java:692)
  1046.         at org.apache.hadoop.hbase.regionserver.RSRpcServices.doNonAtomicRegionMutation(RSRpcServices.java:654)
  1047.         at org.apache.hadoop.hbase.regionserver.RSRpcServices.multi(RSRpcServices.java:2029)
  1048.         at org.apache.hadoop.hbase.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:32213)
  1049.         at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:2112)
  1050.         at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:101)
  1051.         at org.apache.hadoop.hbase.ipc.RpcExecutor.consumerLoop(RpcExecutor.java:130)
  1052.         at org.apache.hadoop.hbase.ipc.RpcExecutor$1.run(RpcExecutor.java:107)
  1053.         at java.lang.Thread.run(Thread.java:745)
  1054.  
  1055.    Locked ownable synchronizers:
  1056.         - None
  1057.  
  1058. "B.defaultRpcServer.handler=45,queue=0,port=9104" #88 daemon prio=5 os_prio=0 tid=0x00007fbf715aa000 nid=0x87 in Object.wait() [0x00007fbf3d1ca000]
  1059.    java.lang.Thread.State: TIMED_WAITING (on object monitor)
  1060.         at java.lang.Object.wait(Native Method)
  1061.         at org.apache.hadoop.hbase.regionserver.wal.SyncFuture.get(SyncFuture.java:167)
  1062.         - locked <0x000000054877e5a8> (a org.apache.hadoop.hbase.regionserver.wal.SyncFuture)
  1063.         at org.apache.hadoop.hbase.regionserver.wal.FSHLog.blockOnSync(FSHLog.java:1472)
  1064.         at org.apache.hadoop.hbase.regionserver.wal.FSHLog.publishSyncThenBlockOnCompletion(FSHLog.java:1466)
  1065.         at org.apache.hadoop.hbase.regionserver.wal.FSHLog.sync(FSHLog.java:1600)
  1066.         at org.apache.hadoop.hbase.regionserver.HRegion.syncOrDefer(HRegion.java:7727)
  1067.         at org.apache.hadoop.hbase.regionserver.HRegion.doMiniBatchMutation(HRegion.java:3145)
  1068.         at org.apache.hadoop.hbase.regionserver.HRegion.batchMutate(HRegion.java:2793)
  1069.         at org.apache.hadoop.hbase.regionserver.HRegion.batchMutate(HRegion.java:2735)
  1070.         at org.apache.hadoop.hbase.regionserver.RSRpcServices.doBatchOp(RSRpcServices.java:692)
  1071.         at org.apache.hadoop.hbase.regionserver.RSRpcServices.doNonAtomicRegionMutation(RSRpcServices.java:654)
  1072.         at org.apache.hadoop.hbase.regionserver.RSRpcServices.multi(RSRpcServices.java:2029)
  1073.         at org.apache.hadoop.hbase.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:32213)
  1074.         at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:2112)
  1075.         at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:101)
  1076.         at org.apache.hadoop.hbase.ipc.RpcExecutor.consumerLoop(RpcExecutor.java:130)
  1077.         at org.apache.hadoop.hbase.ipc.RpcExecutor$1.run(RpcExecutor.java:107)
  1078.         at java.lang.Thread.run(Thread.java:745)
  1079.  
  1080.    Locked ownable synchronizers:
  1081.         - None
  1082.  
  1083. "B.defaultRpcServer.handler=44,queue=4,port=9104" #87 daemon prio=5 os_prio=0 tid=0x00007fbf715a8000 nid=0x86 in Object.wait() [0x00007fbf3d2cb000]
  1084.    java.lang.Thread.State: TIMED_WAITING (on object monitor)
  1085.         at java.lang.Object.wait(Native Method)
  1086.         at org.apache.hadoop.hbase.regionserver.wal.SyncFuture.get(SyncFuture.java:167)
  1087.         - locked <0x000000054893c030> (a org.apache.hadoop.hbase.regionserver.wal.SyncFuture)
  1088.         at org.apache.hadoop.hbase.regionserver.wal.FSHLog.blockOnSync(FSHLog.java:1472)
  1089.         at org.apache.hadoop.hbase.regionserver.wal.FSHLog.publishSyncThenBlockOnCompletion(FSHLog.java:1466)
  1090.         at org.apache.hadoop.hbase.regionserver.wal.FSHLog.sync(FSHLog.java:1600)
  1091.         at org.apache.hadoop.hbase.regionserver.HRegion.syncOrDefer(HRegion.java:7727)
  1092.         at org.apache.hadoop.hbase.regionserver.HRegion.doMiniBatchMutation(HRegion.java:3145)
  1093.         at org.apache.hadoop.hbase.regionserver.HRegion.batchMutate(HRegion.java:2793)
  1094.         at org.apache.hadoop.hbase.regionserver.HRegion.batchMutate(HRegion.java:2735)
  1095.         at org.apache.hadoop.hbase.regionserver.RSRpcServices.doBatchOp(RSRpcServices.java:692)
  1096.         at org.apache.hadoop.hbase.regionserver.RSRpcServices.doNonAtomicRegionMutation(RSRpcServices.java:654)
  1097.         at org.apache.hadoop.hbase.regionserver.RSRpcServices.multi(RSRpcServices.java:2029)
  1098.         at org.apache.hadoop.hbase.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:32213)
  1099.         at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:2112)
  1100.         at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:101)
  1101.         at org.apache.hadoop.hbase.ipc.RpcExecutor.consumerLoop(RpcExecutor.java:130)
  1102.         at org.apache.hadoop.hbase.ipc.RpcExecutor$1.run(RpcExecutor.java:107)
  1103.         at java.lang.Thread.run(Thread.java:745)
  1104.  
  1105.    Locked ownable synchronizers:
  1106.         - None
  1107.  
  1108. "B.defaultRpcServer.handler=43,queue=3,port=9104" #86 daemon prio=5 os_prio=0 tid=0x00007fbf715a6000 nid=0x85 in Object.wait() [0x00007fbf3d3cc000]
  1109.    java.lang.Thread.State: TIMED_WAITING (on object monitor)
  1110.         at java.lang.Object.wait(Native Method)
  1111.         at org.apache.hadoop.hbase.regionserver.wal.SyncFuture.get(SyncFuture.java:167)
  1112.         - locked <0x00000005483009d0> (a org.apache.hadoop.hbase.regionserver.wal.SyncFuture)
  1113.         at org.apache.hadoop.hbase.regionserver.wal.FSHLog.blockOnSync(FSHLog.java:1472)
  1114.         at org.apache.hadoop.hbase.regionserver.wal.FSHLog.publishSyncThenBlockOnCompletion(FSHLog.java:1466)
  1115.         at org.apache.hadoop.hbase.regionserver.wal.FSHLog.sync(FSHLog.java:1600)
  1116.         at org.apache.hadoop.hbase.regionserver.HRegion.syncOrDefer(HRegion.java:7727)
  1117.         at org.apache.hadoop.hbase.regionserver.HRegion.doMiniBatchMutation(HRegion.java:3145)
  1118.         at org.apache.hadoop.hbase.regionserver.HRegion.batchMutate(HRegion.java:2793)
  1119.         at org.apache.hadoop.hbase.regionserver.HRegion.batchMutate(HRegion.java:2735)
  1120.         at org.apache.hadoop.hbase.regionserver.RSRpcServices.doBatchOp(RSRpcServices.java:692)
  1121.         at org.apache.hadoop.hbase.regionserver.RSRpcServices.doNonAtomicRegionMutation(RSRpcServices.java:654)
  1122.         at org.apache.hadoop.hbase.regionserver.RSRpcServices.multi(RSRpcServices.java:2029)
  1123.         at org.apache.hadoop.hbase.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:32213)
  1124.         at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:2112)
  1125.         at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:101)
  1126.         at org.apache.hadoop.hbase.ipc.RpcExecutor.consumerLoop(RpcExecutor.java:130)
  1127.         at org.apache.hadoop.hbase.ipc.RpcExecutor$1.run(RpcExecutor.java:107)
  1128.         at java.lang.Thread.run(Thread.java:745)
  1129.  
  1130.    Locked ownable synchronizers:
  1131.         - None
  1132.  
  1133. "B.defaultRpcServer.handler=42,queue=2,port=9104" #85 daemon prio=5 os_prio=0 tid=0x00007fbf715a4800 nid=0x84 in Object.wait() [0x00007fbf3d4cd000]
  1134.    java.lang.Thread.State: TIMED_WAITING (on object monitor)
  1135.         at java.lang.Object.wait(Native Method)
  1136.         at org.apache.hadoop.hbase.regionserver.wal.SyncFuture.get(SyncFuture.java:167)
  1137.         - locked <0x000000054877e558> (a org.apache.hadoop.hbase.regionserver.wal.SyncFuture)
  1138.         at org.apache.hadoop.hbase.regionserver.wal.FSHLog.blockOnSync(FSHLog.java:1472)
  1139.         at org.apache.hadoop.hbase.regionserver.wal.FSHLog.publishSyncThenBlockOnCompletion(FSHLog.java:1466)
  1140.         at org.apache.hadoop.hbase.regionserver.wal.FSHLog.sync(FSHLog.java:1600)
  1141.         at org.apache.hadoop.hbase.regionserver.HRegion.syncOrDefer(HRegion.java:7727)
  1142.         at org.apache.hadoop.hbase.regionserver.HRegion.doMiniBatchMutation(HRegion.java:3145)
  1143.         at org.apache.hadoop.hbase.regionserver.HRegion.batchMutate(HRegion.java:2793)
  1144.         at org.apache.hadoop.hbase.regionserver.HRegion.batchMutate(HRegion.java:2735)
  1145.         at org.apache.hadoop.hbase.regionserver.RSRpcServices.doBatchOp(RSRpcServices.java:692)
  1146.         at org.apache.hadoop.hbase.regionserver.RSRpcServices.doNonAtomicRegionMutation(RSRpcServices.java:654)
  1147.         at org.apache.hadoop.hbase.regionserver.RSRpcServices.multi(RSRpcServices.java:2029)
  1148.         at org.apache.hadoop.hbase.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:32213)
  1149.         at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:2112)
  1150.         at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:101)
  1151.         at org.apache.hadoop.hbase.ipc.RpcExecutor.consumerLoop(RpcExecutor.java:130)
  1152.         at org.apache.hadoop.hbase.ipc.RpcExecutor$1.run(RpcExecutor.java:107)
  1153.         at java.lang.Thread.run(Thread.java:745)
  1154.  
  1155.    Locked ownable synchronizers:
  1156.         - None
  1157.  
  1158. "B.defaultRpcServer.handler=41,queue=1,port=9104" #84 daemon prio=5 os_prio=0 tid=0x00007fbf715a2000 nid=0x83 in Object.wait() [0x00007fbf3d5ce000]
  1159.    java.lang.Thread.State: TIMED_WAITING (on object monitor)
  1160.         at java.lang.Object.wait(Native Method)
  1161.         at org.apache.hadoop.hbase.regionserver.wal.SyncFuture.get(SyncFuture.java:167)
  1162.         - locked <0x0000000548c50de0> (a org.apache.hadoop.hbase.regionserver.wal.SyncFuture)
  1163.         at org.apache.hadoop.hbase.regionserver.wal.FSHLog.blockOnSync(FSHLog.java:1472)
  1164.         at org.apache.hadoop.hbase.regionserver.wal.FSHLog.publishSyncThenBlockOnCompletion(FSHLog.java:1466)
  1165.         at org.apache.hadoop.hbase.regionserver.wal.FSHLog.sync(FSHLog.java:1600)
  1166.         at org.apache.hadoop.hbase.regionserver.HRegion.syncOrDefer(HRegion.java:7727)
  1167.         at org.apache.hadoop.hbase.regionserver.HRegion.doMiniBatchMutation(HRegion.java:3145)
  1168.         at org.apache.hadoop.hbase.regionserver.HRegion.batchMutate(HRegion.java:2793)
  1169.         at org.apache.hadoop.hbase.regionserver.HRegion.batchMutate(HRegion.java:2735)
  1170.         at org.apache.hadoop.hbase.regionserver.RSRpcServices.doBatchOp(RSRpcServices.java:692)
  1171.         at org.apache.hadoop.hbase.regionserver.RSRpcServices.doNonAtomicRegionMutation(RSRpcServices.java:654)
  1172.         at org.apache.hadoop.hbase.regionserver.RSRpcServices.multi(RSRpcServices.java:2029)
  1173.         at org.apache.hadoop.hbase.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:32213)
  1174.         at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:2112)
  1175.         at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:101)
  1176.         at org.apache.hadoop.hbase.ipc.RpcExecutor.consumerLoop(RpcExecutor.java:130)
  1177.         at org.apache.hadoop.hbase.ipc.RpcExecutor$1.run(RpcExecutor.java:107)
  1178.         at java.lang.Thread.run(Thread.java:745)
  1179.  
  1180.    Locked ownable synchronizers:
  1181.         - None
  1182.  
  1183. "B.defaultRpcServer.handler=40,queue=0,port=9104" #83 daemon prio=5 os_prio=0 tid=0x00007fbf715a0000 nid=0x82 in Object.wait() [0x00007fbf3d6cf000]
  1184.    java.lang.Thread.State: TIMED_WAITING (on object monitor)
  1185.         at java.lang.Object.wait(Native Method)
  1186.         at org.apache.hadoop.hbase.regionserver.wal.SyncFuture.get(SyncFuture.java:167)
  1187.         - locked <0x0000000548a3cea8> (a org.apache.hadoop.hbase.regionserver.wal.SyncFuture)
  1188.         at org.apache.hadoop.hbase.regionserver.wal.FSHLog.blockOnSync(FSHLog.java:1472)
  1189.         at org.apache.hadoop.hbase.regionserver.wal.FSHLog.publishSyncThenBlockOnCompletion(FSHLog.java:1466)
  1190.         at org.apache.hadoop.hbase.regionserver.wal.FSHLog.sync(FSHLog.java:1600)
  1191.         at org.apache.hadoop.hbase.regionserver.HRegion.syncOrDefer(HRegion.java:7727)
  1192.         at org.apache.hadoop.hbase.regionserver.HRegion.doMiniBatchMutation(HRegion.java:3145)
  1193.         at org.apache.hadoop.hbase.regionserver.HRegion.batchMutate(HRegion.java:2793)
  1194.         at org.apache.hadoop.hbase.regionserver.HRegion.batchMutate(HRegion.java:2735)
  1195.         at org.apache.hadoop.hbase.regionserver.RSRpcServices.doBatchOp(RSRpcServices.java:692)
  1196.         at org.apache.hadoop.hbase.regionserver.RSRpcServices.doNonAtomicRegionMutation(RSRpcServices.java:654)
  1197.         at org.apache.hadoop.hbase.regionserver.RSRpcServices.multi(RSRpcServices.java:2029)
  1198.         at org.apache.hadoop.hbase.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:32213)
  1199.         at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:2112)
  1200.         at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:101)
  1201.         at org.apache.hadoop.hbase.ipc.RpcExecutor.consumerLoop(RpcExecutor.java:130)
  1202.         at org.apache.hadoop.hbase.ipc.RpcExecutor$1.run(RpcExecutor.java:107)
  1203.         at java.lang.Thread.run(Thread.java:745)
  1204.  
  1205.    Locked ownable synchronizers:
  1206.         - None
  1207.  
  1208. "B.defaultRpcServer.handler=39,queue=4,port=9104" #82 daemon prio=5 os_prio=0 tid=0x00007fbf7159e800 nid=0x81 in Object.wait() [0x00007fbf3d7d0000]
  1209.    java.lang.Thread.State: TIMED_WAITING (on object monitor)
  1210.         at java.lang.Object.wait(Native Method)
  1211.         at org.apache.hadoop.hbase.regionserver.wal.SyncFuture.get(SyncFuture.java:167)
  1212.         - locked <0x000000054893c198> (a org.apache.hadoop.hbase.regionserver.wal.SyncFuture)
  1213.         at org.apache.hadoop.hbase.regionserver.wal.FSHLog.blockOnSync(FSHLog.java:1472)
  1214.         at org.apache.hadoop.hbase.regionserver.wal.FSHLog.publishSyncThenBlockOnCompletion(FSHLog.java:1466)
  1215.         at org.apache.hadoop.hbase.regionserver.wal.FSHLog.sync(FSHLog.java:1600)
  1216.         at org.apache.hadoop.hbase.regionserver.HRegion.syncOrDefer(HRegion.java:7727)
  1217.         at org.apache.hadoop.hbase.regionserver.HRegion.doMiniBatchMutation(HRegion.java:3145)
  1218.         at org.apache.hadoop.hbase.regionserver.HRegion.batchMutate(HRegion.java:2793)
  1219.         at org.apache.hadoop.hbase.regionserver.HRegion.batchMutate(HRegion.java:2735)
  1220.         at org.apache.hadoop.hbase.regionserver.RSRpcServices.doBatchOp(RSRpcServices.java:692)
  1221.         at org.apache.hadoop.hbase.regionserver.RSRpcServices.doNonAtomicRegionMutation(RSRpcServices.java:654)
  1222.         at org.apache.hadoop.hbase.regionserver.RSRpcServices.multi(RSRpcServices.java:2029)
  1223.         at org.apache.hadoop.hbase.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:32213)
  1224.         at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:2112)
  1225.         at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:101)
  1226.         at org.apache.hadoop.hbase.ipc.RpcExecutor.consumerLoop(RpcExecutor.java:130)
  1227.         at org.apache.hadoop.hbase.ipc.RpcExecutor$1.run(RpcExecutor.java:107)
  1228.         at java.lang.Thread.run(Thread.java:745)
  1229.  
  1230.    Locked ownable synchronizers:
  1231.         - None
  1232.  
  1233. "B.defaultRpcServer.handler=38,queue=3,port=9104" #81 daemon prio=5 os_prio=0 tid=0x00007fbf7159c800 nid=0x80 in Object.wait() [0x00007fbf3d8d1000]
  1234.    java.lang.Thread.State: TIMED_WAITING (on object monitor)
  1235.         at java.lang.Object.wait(Native Method)
  1236.         at org.apache.hadoop.hbase.regionserver.wal.SyncFuture.get(SyncFuture.java:167)
  1237.         - locked <0x0000000548f93eb0> (a org.apache.hadoop.hbase.regionserver.wal.SyncFuture)
  1238.         at org.apache.hadoop.hbase.regionserver.wal.FSHLog.blockOnSync(FSHLog.java:1472)
  1239.         at org.apache.hadoop.hbase.regionserver.wal.FSHLog.publishSyncThenBlockOnCompletion(FSHLog.java:1466)
  1240.         at org.apache.hadoop.hbase.regionserver.wal.FSHLog.sync(FSHLog.java:1600)
  1241.         at org.apache.hadoop.hbase.regionserver.HRegion.syncOrDefer(HRegion.java:7727)
  1242.         at org.apache.hadoop.hbase.regionserver.HRegion.doMiniBatchMutation(HRegion.java:3145)
  1243.         at org.apache.hadoop.hbase.regionserver.HRegion.batchMutate(HRegion.java:2793)
  1244.         at org.apache.hadoop.hbase.regionserver.HRegion.batchMutate(HRegion.java:2735)
  1245.         at org.apache.hadoop.hbase.regionserver.RSRpcServices.doBatchOp(RSRpcServices.java:692)
  1246.         at org.apache.hadoop.hbase.regionserver.RSRpcServices.doNonAtomicRegionMutation(RSRpcServices.java:654)
  1247.         at org.apache.hadoop.hbase.regionserver.RSRpcServices.multi(RSRpcServices.java:2029)
  1248.         at org.apache.hadoop.hbase.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:32213)
  1249.         at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:2112)
  1250.         at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:101)
  1251.         at org.apache.hadoop.hbase.ipc.RpcExecutor.consumerLoop(RpcExecutor.java:130)
  1252.         at org.apache.hadoop.hbase.ipc.RpcExecutor$1.run(RpcExecutor.java:107)
  1253.         at java.lang.Thread.run(Thread.java:745)
  1254.  
  1255.    Locked ownable synchronizers:
  1256.         - None
  1257.  
  1258. "B.defaultRpcServer.handler=37,queue=2,port=9104" #80 daemon prio=5 os_prio=0 tid=0x00007fbf7159a800 nid=0x7f in Object.wait() [0x00007fbf3d9d2000]
  1259.    java.lang.Thread.State: TIMED_WAITING (on object monitor)
  1260.         at java.lang.Object.wait(Native Method)
  1261.         at org.apache.hadoop.hbase.regionserver.wal.SyncFuture.get(SyncFuture.java:167)
  1262.         - locked <0x0000000548f94248> (a org.apache.hadoop.hbase.regionserver.wal.SyncFuture)
  1263.         at org.apache.hadoop.hbase.regionserver.wal.FSHLog.blockOnSync(FSHLog.java:1472)
  1264.         at org.apache.hadoop.hbase.regionserver.wal.FSHLog.publishSyncThenBlockOnCompletion(FSHLog.java:1466)
  1265.         at org.apache.hadoop.hbase.regionserver.wal.FSHLog.sync(FSHLog.java:1600)
  1266.         at org.apache.hadoop.hbase.regionserver.HRegion.syncOrDefer(HRegion.java:7727)
  1267.         at org.apache.hadoop.hbase.regionserver.HRegion.doMiniBatchMutation(HRegion.java:3145)
  1268.         at org.apache.hadoop.hbase.regionserver.HRegion.batchMutate(HRegion.java:2793)
  1269.         at org.apache.hadoop.hbase.regionserver.HRegion.batchMutate(HRegion.java:2735)
  1270.         at org.apache.hadoop.hbase.regionserver.RSRpcServices.doBatchOp(RSRpcServices.java:692)
  1271.         at org.apache.hadoop.hbase.regionserver.RSRpcServices.doNonAtomicRegionMutation(RSRpcServices.java:654)
  1272.         at org.apache.hadoop.hbase.regionserver.RSRpcServices.multi(RSRpcServices.java:2029)
  1273.         at org.apache.hadoop.hbase.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:32213)
  1274.         at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:2112)
  1275.         at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:101)
  1276.         at org.apache.hadoop.hbase.ipc.RpcExecutor.consumerLoop(RpcExecutor.java:130)
  1277.         at org.apache.hadoop.hbase.ipc.RpcExecutor$1.run(RpcExecutor.java:107)
  1278.         at java.lang.Thread.run(Thread.java:745)
  1279.  
  1280.    Locked ownable synchronizers:
  1281.         - None
  1282.  
  1283. "B.defaultRpcServer.handler=36,queue=1,port=9104" #79 daemon prio=5 os_prio=0 tid=0x00007fbf71599000 nid=0x7e in Object.wait() [0x00007fbf3dad3000]
  1284.    java.lang.Thread.State: TIMED_WAITING (on object monitor)
  1285.         at java.lang.Object.wait(Native Method)
  1286.         at org.apache.hadoop.hbase.regionserver.wal.SyncFuture.get(SyncFuture.java:167)
  1287.         - locked <0x0000000548f7efc8> (a org.apache.hadoop.hbase.regionserver.wal.SyncFuture)
  1288.         at org.apache.hadoop.hbase.regionserver.wal.FSHLog.blockOnSync(FSHLog.java:1472)
  1289.         at org.apache.hadoop.hbase.regionserver.wal.FSHLog.publishSyncThenBlockOnCompletion(FSHLog.java:1466)
  1290.         at org.apache.hadoop.hbase.regionserver.wal.FSHLog.sync(FSHLog.java:1600)
  1291.         at org.apache.hadoop.hbase.regionserver.HRegion.syncOrDefer(HRegion.java:7727)
  1292.         at org.apache.hadoop.hbase.regionserver.HRegion.doMiniBatchMutation(HRegion.java:3145)
  1293.         at org.apache.hadoop.hbase.regionserver.HRegion.batchMutate(HRegion.java:2793)
  1294.         at org.apache.hadoop.hbase.regionserver.HRegion.batchMutate(HRegion.java:2735)
  1295.         at org.apache.hadoop.hbase.regionserver.RSRpcServices.doBatchOp(RSRpcServices.java:692)
  1296.         at org.apache.hadoop.hbase.regionserver.RSRpcServices.doNonAtomicRegionMutation(RSRpcServices.java:654)
  1297.         at org.apache.hadoop.hbase.regionserver.RSRpcServices.multi(RSRpcServices.java:2029)
  1298.         at org.apache.hadoop.hbase.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:32213)
  1299.         at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:2112)
  1300.         at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:101)
  1301.         at org.apache.hadoop.hbase.ipc.RpcExecutor.consumerLoop(RpcExecutor.java:130)
  1302.         at org.apache.hadoop.hbase.ipc.RpcExecutor$1.run(RpcExecutor.java:107)
  1303.         at java.lang.Thread.run(Thread.java:745)
  1304.  
  1305.    Locked ownable synchronizers:
  1306.         - None
  1307.  
  1308. "B.defaultRpcServer.handler=35,queue=0,port=9104" #78 daemon prio=5 os_prio=0 tid=0x00007fbf71549000 nid=0x7d in Object.wait() [0x00007fbf3dbd4000]
  1309.    java.lang.Thread.State: TIMED_WAITING (on object monitor)
  1310.         at java.lang.Object.wait(Native Method)
  1311.         at org.apache.hadoop.hbase.regionserver.wal.SyncFuture.get(SyncFuture.java:167)
  1312.         - locked <0x0000000548b22e10> (a org.apache.hadoop.hbase.regionserver.wal.SyncFuture)
  1313.         at org.apache.hadoop.hbase.regionserver.wal.FSHLog.blockOnSync(FSHLog.java:1472)
  1314.         at org.apache.hadoop.hbase.regionserver.wal.FSHLog.publishSyncThenBlockOnCompletion(FSHLog.java:1466)
  1315.         at org.apache.hadoop.hbase.regionserver.wal.FSHLog.sync(FSHLog.java:1600)
  1316.         at org.apache.hadoop.hbase.regionserver.HRegion.syncOrDefer(HRegion.java:7727)
  1317.         at org.apache.hadoop.hbase.regionserver.HRegion.doMiniBatchMutation(HRegion.java:3145)
  1318.         at org.apache.hadoop.hbase.regionserver.HRegion.batchMutate(HRegion.java:2793)
  1319.         at org.apache.hadoop.hbase.regionserver.HRegion.batchMutate(HRegion.java:2735)
  1320.         at org.apache.hadoop.hbase.regionserver.RSRpcServices.doBatchOp(RSRpcServices.java:692)
  1321.         at org.apache.hadoop.hbase.regionserver.RSRpcServices.doNonAtomicRegionMutation(RSRpcServices.java:654)
  1322.         at org.apache.hadoop.hbase.regionserver.RSRpcServices.multi(RSRpcServices.java:2029)
  1323.         at org.apache.hadoop.hbase.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:32213)
  1324.         at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:2112)
  1325.         at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:101)
  1326.         at org.apache.hadoop.hbase.ipc.RpcExecutor.consumerLoop(RpcExecutor.java:130)
  1327.         at org.apache.hadoop.hbase.ipc.RpcExecutor$1.run(RpcExecutor.java:107)
  1328.         at java.lang.Thread.run(Thread.java:745)
  1329.  
  1330.    Locked ownable synchronizers:
  1331.         - None
  1332.  
  1333. "B.defaultRpcServer.handler=34,queue=4,port=9104" #77 daemon prio=5 os_prio=0 tid=0x00007fbf71547000 nid=0x7c in Object.wait() [0x00007fbf3dcd5000]
  1334.    java.lang.Thread.State: TIMED_WAITING (on object monitor)
  1335.         at java.lang.Object.wait(Native Method)
  1336.         at org.apache.hadoop.hbase.regionserver.wal.SyncFuture.get(SyncFuture.java:167)
  1337.         - locked <0x0000000548c9a180> (a org.apache.hadoop.hbase.regionserver.wal.SyncFuture)
  1338.         at org.apache.hadoop.hbase.regionserver.wal.FSHLog.blockOnSync(FSHLog.java:1472)
  1339.         at org.apache.hadoop.hbase.regionserver.wal.FSHLog.publishSyncThenBlockOnCompletion(FSHLog.java:1466)
  1340.         at org.apache.hadoop.hbase.regionserver.wal.FSHLog.sync(FSHLog.java:1600)
  1341.         at org.apache.hadoop.hbase.regionserver.HRegion.syncOrDefer(HRegion.java:7727)
  1342.         at org.apache.hadoop.hbase.regionserver.HRegion.doMiniBatchMutation(HRegion.java:3145)
  1343.         at org.apache.hadoop.hbase.regionserver.HRegion.batchMutate(HRegion.java:2793)
  1344.         at org.apache.hadoop.hbase.regionserver.HRegion.batchMutate(HRegion.java:2735)
  1345.         at org.apache.hadoop.hbase.regionserver.RSRpcServices.doBatchOp(RSRpcServices.java:692)
  1346.         at org.apache.hadoop.hbase.regionserver.RSRpcServices.doNonAtomicRegionMutation(RSRpcServices.java:654)
  1347.         at org.apache.hadoop.hbase.regionserver.RSRpcServices.multi(RSRpcServices.java:2029)
  1348.         at org.apache.hadoop.hbase.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:32213)
  1349.         at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:2112)
  1350.         at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:101)
  1351.         at org.apache.hadoop.hbase.ipc.RpcExecutor.consumerLoop(RpcExecutor.java:130)
  1352.         at org.apache.hadoop.hbase.ipc.RpcExecutor$1.run(RpcExecutor.java:107)
  1353.         at java.lang.Thread.run(Thread.java:745)
  1354.  
  1355.    Locked ownable synchronizers:
  1356.         - None
  1357.  
  1358. "B.defaultRpcServer.handler=33,queue=3,port=9104" #76 daemon prio=5 os_prio=0 tid=0x00007fbf71545000 nid=0x7b in Object.wait() [0x00007fbf3ddd6000]
  1359.    java.lang.Thread.State: TIMED_WAITING (on object monitor)
  1360.         at java.lang.Object.wait(Native Method)
  1361.         at org.apache.hadoop.hbase.regionserver.wal.SyncFuture.get(SyncFuture.java:167)
  1362.         - locked <0x00000005486e9f10> (a org.apache.hadoop.hbase.regionserver.wal.SyncFuture)
  1363.         at org.apache.hadoop.hbase.regionserver.wal.FSHLog.blockOnSync(FSHLog.java:1472)
  1364.         at org.apache.hadoop.hbase.regionserver.wal.FSHLog.publishSyncThenBlockOnCompletion(FSHLog.java:1466)
  1365.         at org.apache.hadoop.hbase.regionserver.wal.FSHLog.sync(FSHLog.java:1600)
  1366.         at org.apache.hadoop.hbase.regionserver.HRegion.syncOrDefer(HRegion.java:7727)
  1367.         at org.apache.hadoop.hbase.regionserver.HRegion.doMiniBatchMutation(HRegion.java:3145)
  1368.         at org.apache.hadoop.hbase.regionserver.HRegion.batchMutate(HRegion.java:2793)
  1369.         at org.apache.hadoop.hbase.regionserver.HRegion.batchMutate(HRegion.java:2735)
  1370.         at org.apache.hadoop.hbase.regionserver.RSRpcServices.doBatchOp(RSRpcServices.java:692)
  1371.         at org.apache.hadoop.hbase.regionserver.RSRpcServices.doNonAtomicRegionMutation(RSRpcServices.java:654)
  1372.         at org.apache.hadoop.hbase.regionserver.RSRpcServices.multi(RSRpcServices.java:2029)
  1373.         at org.apache.hadoop.hbase.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:32213)
  1374.         at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:2112)
  1375.         at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:101)
  1376.         at org.apache.hadoop.hbase.ipc.RpcExecutor.consumerLoop(RpcExecutor.java:130)
  1377.         at org.apache.hadoop.hbase.ipc.RpcExecutor$1.run(RpcExecutor.java:107)
  1378.         at java.lang.Thread.run(Thread.java:745)
  1379.  
  1380.    Locked ownable synchronizers:
  1381.         - None
  1382.  
  1383. "B.defaultRpcServer.handler=32,queue=2,port=9104" #75 daemon prio=5 os_prio=0 tid=0x00007fbf71543000 nid=0x7a in Object.wait() [0x00007fbf3ded7000]
  1384.    java.lang.Thread.State: TIMED_WAITING (on object monitor)
  1385.         at java.lang.Object.wait(Native Method)
  1386.         at org.apache.hadoop.hbase.regionserver.wal.SyncFuture.get(SyncFuture.java:167)
  1387.         - locked <0x0000000548c51088> (a org.apache.hadoop.hbase.regionserver.wal.SyncFuture)
  1388.         at org.apache.hadoop.hbase.regionserver.wal.FSHLog.blockOnSync(FSHLog.java:1472)
  1389.         at org.apache.hadoop.hbase.regionserver.wal.FSHLog.publishSyncThenBlockOnCompletion(FSHLog.java:1466)
  1390.         at org.apache.hadoop.hbase.regionserver.wal.FSHLog.sync(FSHLog.java:1600)
  1391.         at org.apache.hadoop.hbase.regionserver.HRegion.syncOrDefer(HRegion.java:7727)
  1392.         at org.apache.hadoop.hbase.regionserver.HRegion.doMiniBatchMutation(HRegion.java:3145)
  1393.         at org.apache.hadoop.hbase.regionserver.HRegion.batchMutate(HRegion.java:2793)
  1394.         at org.apache.hadoop.hbase.regionserver.HRegion.batchMutate(HRegion.java:2735)
  1395.         at org.apache.hadoop.hbase.regionserver.RSRpcServices.doBatchOp(RSRpcServices.java:692)
  1396.         at org.apache.hadoop.hbase.regionserver.RSRpcServices.doNonAtomicRegionMutation(RSRpcServices.java:654)
  1397.         at org.apache.hadoop.hbase.regionserver.RSRpcServices.multi(RSRpcServices.java:2029)
  1398.         at org.apache.hadoop.hbase.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:32213)
  1399.         at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:2112)
  1400.         at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:101)
  1401.         at org.apache.hadoop.hbase.ipc.RpcExecutor.consumerLoop(RpcExecutor.java:130)
  1402.         at org.apache.hadoop.hbase.ipc.RpcExecutor$1.run(RpcExecutor.java:107)
  1403.         at java.lang.Thread.run(Thread.java:745)
  1404.  
  1405.    Locked ownable synchronizers:
  1406.         - None
  1407.  
  1408. "B.defaultRpcServer.handler=31,queue=1,port=9104" #74 daemon prio=5 os_prio=0 tid=0x00007fbf71541000 nid=0x79 in Object.wait() [0x00007fbf3dfd8000]
  1409.    java.lang.Thread.State: TIMED_WAITING (on object monitor)
  1410.         at java.lang.Object.wait(Native Method)
  1411.         at org.apache.hadoop.hbase.regionserver.wal.SyncFuture.get(SyncFuture.java:167)
  1412.         - locked <0x0000000548d42c60> (a org.apache.hadoop.hbase.regionserver.wal.SyncFuture)
  1413.         at org.apache.hadoop.hbase.regionserver.wal.FSHLog.blockOnSync(FSHLog.java:1472)
  1414.         at org.apache.hadoop.hbase.regionserver.wal.FSHLog.publishSyncThenBlockOnCompletion(FSHLog.java:1466)
  1415.         at org.apache.hadoop.hbase.regionserver.wal.FSHLog.sync(FSHLog.java:1600)
  1416.         at org.apache.hadoop.hbase.regionserver.HRegion.syncOrDefer(HRegion.java:7727)
  1417.         at org.apache.hadoop.hbase.regionserver.HRegion.doMiniBatchMutation(HRegion.java:3145)
  1418.         at org.apache.hadoop.hbase.regionserver.HRegion.batchMutate(HRegion.java:2793)
  1419.         at org.apache.hadoop.hbase.regionserver.HRegion.batchMutate(HRegion.java:2735)
  1420.         at org.apache.hadoop.hbase.regionserver.RSRpcServices.doBatchOp(RSRpcServices.java:692)
  1421.         at org.apache.hadoop.hbase.regionserver.RSRpcServices.doNonAtomicRegionMutation(RSRpcServices.java:654)
  1422.         at org.apache.hadoop.hbase.regionserver.RSRpcServices.multi(RSRpcServices.java:2029)
  1423.         at org.apache.hadoop.hbase.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:32213)
  1424.         at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:2112)
  1425.         at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:101)
  1426.         at org.apache.hadoop.hbase.ipc.RpcExecutor.consumerLoop(RpcExecutor.java:130)
  1427.         at org.apache.hadoop.hbase.ipc.RpcExecutor$1.run(RpcExecutor.java:107)
  1428.         at java.lang.Thread.run(Thread.java:745)
  1429.  
  1430.    Locked ownable synchronizers:
  1431.         - None
  1432.  
  1433. "B.defaultRpcServer.handler=30,queue=0,port=9104" #73 daemon prio=5 os_prio=0 tid=0x00007fbf7153f000 nid=0x78 in Object.wait() [0x00007fbf3e0d9000]
  1434.    java.lang.Thread.State: TIMED_WAITING (on object monitor)
  1435.         at java.lang.Object.wait(Native Method)
  1436.         at org.apache.hadoop.hbase.regionserver.wal.SyncFuture.get(SyncFuture.java:167)
  1437.         - locked <0x0000000548b9b8d0> (a org.apache.hadoop.hbase.regionserver.wal.SyncFuture)
  1438.         at org.apache.hadoop.hbase.regionserver.wal.FSHLog.blockOnSync(FSHLog.java:1472)
  1439.         at org.apache.hadoop.hbase.regionserver.wal.FSHLog.publishSyncThenBlockOnCompletion(FSHLog.java:1466)
  1440.         at org.apache.hadoop.hbase.regionserver.wal.FSHLog.sync(FSHLog.java:1600)
  1441.         at org.apache.hadoop.hbase.regionserver.HRegion.syncOrDefer(HRegion.java:7727)
  1442.         at org.apache.hadoop.hbase.regionserver.HRegion.doMiniBatchMutation(HRegion.java:3145)
  1443.         at org.apache.hadoop.hbase.regionserver.HRegion.batchMutate(HRegion.java:2793)
  1444.         at org.apache.hadoop.hbase.regionserver.HRegion.batchMutate(HRegion.java:2735)
  1445.         at org.apache.hadoop.hbase.regionserver.RSRpcServices.doBatchOp(RSRpcServices.java:692)
  1446.         at org.apache.hadoop.hbase.regionserver.RSRpcServices.doNonAtomicRegionMutation(RSRpcServices.java:654)
  1447.         at org.apache.hadoop.hbase.regionserver.RSRpcServices.multi(RSRpcServices.java:2029)
  1448.         at org.apache.hadoop.hbase.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:32213)
  1449.         at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:2112)
  1450.         at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:101)
  1451.         at org.apache.hadoop.hbase.ipc.RpcExecutor.consumerLoop(RpcExecutor.java:130)
  1452.         at org.apache.hadoop.hbase.ipc.RpcExecutor$1.run(RpcExecutor.java:107)
  1453.         at java.lang.Thread.run(Thread.java:745)
  1454.  
  1455.    Locked ownable synchronizers:
  1456.         - None
  1457.  
  1458. "B.defaultRpcServer.handler=29,queue=4,port=9104" #72 daemon prio=5 os_prio=0 tid=0x00007fbf7153d000 nid=0x77 in Object.wait() [0x00007fbf3e1da000]
  1459.    java.lang.Thread.State: TIMED_WAITING (on object monitor)
  1460.         at java.lang.Object.wait(Native Method)
  1461.         at org.apache.hadoop.hbase.regionserver.wal.SyncFuture.get(SyncFuture.java:167)
  1462.         - locked <0x000000054893c1e8> (a org.apache.hadoop.hbase.regionserver.wal.SyncFuture)
  1463.         at org.apache.hadoop.hbase.regionserver.wal.FSHLog.blockOnSync(FSHLog.java:1472)
  1464.         at org.apache.hadoop.hbase.regionserver.wal.FSHLog.publishSyncThenBlockOnCompletion(FSHLog.java:1466)
  1465.         at org.apache.hadoop.hbase.regionserver.wal.FSHLog.sync(FSHLog.java:1600)
  1466.         at org.apache.hadoop.hbase.regionserver.HRegion.syncOrDefer(HRegion.java:7727)
  1467.         at org.apache.hadoop.hbase.regionserver.HRegion.doMiniBatchMutation(HRegion.java:3145)
  1468.         at org.apache.hadoop.hbase.regionserver.HRegion.batchMutate(HRegion.java:2793)
  1469.         at org.apache.hadoop.hbase.regionserver.HRegion.batchMutate(HRegion.java:2735)
  1470.         at org.apache.hadoop.hbase.regionserver.RSRpcServices.doBatchOp(RSRpcServices.java:692)
  1471.         at org.apache.hadoop.hbase.regionserver.RSRpcServices.doNonAtomicRegionMutation(RSRpcServices.java:654)
  1472.         at org.apache.hadoop.hbase.regionserver.RSRpcServices.multi(RSRpcServices.java:2029)
  1473.         at org.apache.hadoop.hbase.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:32213)
  1474.         at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:2112)
  1475.         at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:101)
  1476.         at org.apache.hadoop.hbase.ipc.RpcExecutor.consumerLoop(RpcExecutor.java:130)
  1477.         at org.apache.hadoop.hbase.ipc.RpcExecutor$1.run(RpcExecutor.java:107)
  1478.         at java.lang.Thread.run(Thread.java:745)
  1479.  
  1480.    Locked ownable synchronizers:
  1481.         - None
  1482.  
  1483. "B.defaultRpcServer.handler=28,queue=3,port=9104" #71 daemon prio=5 os_prio=0 tid=0x00007fbf7153b800 nid=0x76 in Object.wait() [0x00007fbf3e2db000]
  1484.    java.lang.Thread.State: TIMED_WAITING (on object monitor)
  1485.         at java.lang.Object.wait(Native Method)
  1486.         at org.apache.hadoop.hbase.regionserver.wal.SyncFuture.get(SyncFuture.java:167)
  1487.         - locked <0x0000000548f93e60> (a org.apache.hadoop.hbase.regionserver.wal.SyncFuture)
  1488.         at org.apache.hadoop.hbase.regionserver.wal.FSHLog.blockOnSync(FSHLog.java:1472)
  1489.         at org.apache.hadoop.hbase.regionserver.wal.FSHLog.publishSyncThenBlockOnCompletion(FSHLog.java:1466)
  1490.         at org.apache.hadoop.hbase.regionserver.wal.FSHLog.sync(FSHLog.java:1600)
  1491.         at org.apache.hadoop.hbase.regionserver.HRegion.syncOrDefer(HRegion.java:7727)
  1492.         at org.apache.hadoop.hbase.regionserver.HRegion.doMiniBatchMutation(HRegion.java:3145)
  1493.         at org.apache.hadoop.hbase.regionserver.HRegion.batchMutate(HRegion.java:2793)
  1494.         at org.apache.hadoop.hbase.regionserver.HRegion.batchMutate(HRegion.java:2735)
  1495.         at org.apache.hadoop.hbase.regionserver.RSRpcServices.doBatchOp(RSRpcServices.java:692)
  1496.         at org.apache.hadoop.hbase.regionserver.RSRpcServices.doNonAtomicRegionMutation(RSRpcServices.java:654)
  1497.         at org.apache.hadoop.hbase.regionserver.RSRpcServices.multi(RSRpcServices.java:2029)
  1498.         at org.apache.hadoop.hbase.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:32213)
  1499.         at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:2112)
  1500.         at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:101)
  1501.         at org.apache.hadoop.hbase.ipc.RpcExecutor.consumerLoop(RpcExecutor.java:130)
  1502.         at org.apache.hadoop.hbase.ipc.RpcExecutor$1.run(RpcExecutor.java:107)
  1503.         at java.lang.Thread.run(Thread.java:745)
  1504.  
  1505.    Locked ownable synchronizers:
  1506.         - None
  1507.  
  1508. "B.defaultRpcServer.handler=27,queue=2,port=9104" #70 daemon prio=5 os_prio=0 tid=0x00007fbf71613000 nid=0x75 in Object.wait() [0x00007fbf3e3dc000]
  1509.    java.lang.Thread.State: TIMED_WAITING (on object monitor)
  1510.         at java.lang.Object.wait(Native Method)
  1511.         at org.apache.hadoop.hbase.regionserver.wal.SyncFuture.get(SyncFuture.java:167)
  1512.         - locked <0x0000000548c510d8> (a org.apache.hadoop.hbase.regionserver.wal.SyncFuture)
  1513.         at org.apache.hadoop.hbase.regionserver.wal.FSHLog.blockOnSync(FSHLog.java:1472)
  1514.         at org.apache.hadoop.hbase.regionserver.wal.FSHLog.publishSyncThenBlockOnCompletion(FSHLog.java:1466)
  1515.         at org.apache.hadoop.hbase.regionserver.wal.FSHLog.sync(FSHLog.java:1600)
  1516.         at org.apache.hadoop.hbase.regionserver.HRegion.syncOrDefer(HRegion.java:7727)
  1517.         at org.apache.hadoop.hbase.regionserver.HRegion.doMiniBatchMutation(HRegion.java:3145)
  1518.         at org.apache.hadoop.hbase.regionserver.HRegion.batchMutate(HRegion.java:2793)
  1519.         at org.apache.hadoop.hbase.regionserver.HRegion.batchMutate(HRegion.java:2735)
  1520.         at org.apache.hadoop.hbase.regionserver.RSRpcServices.doBatchOp(RSRpcServices.java:692)
  1521.         at org.apache.hadoop.hbase.regionserver.RSRpcServices.doNonAtomicRegionMutation(RSRpcServices.java:654)
  1522.         at org.apache.hadoop.hbase.regionserver.RSRpcServices.multi(RSRpcServices.java:2029)
  1523.         at org.apache.hadoop.hbase.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:32213)
  1524.         at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:2112)
  1525.         at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:101)
  1526.         at org.apache.hadoop.hbase.ipc.RpcExecutor.consumerLoop(RpcExecutor.java:130)
  1527.         at org.apache.hadoop.hbase.ipc.RpcExecutor$1.run(RpcExecutor.java:107)
  1528.         at java.lang.Thread.run(Thread.java:745)
  1529.  
  1530.    Locked ownable synchronizers:
  1531.         - None
  1532.  
  1533. "B.defaultRpcServer.handler=26,queue=1,port=9104" #69 daemon prio=5 os_prio=0 tid=0x00007fbf71611800 nid=0x74 in Object.wait() [0x00007fbf3e4dd000]
  1534.    java.lang.Thread.State: TIMED_WAITING (on object monitor)
  1535.         at java.lang.Object.wait(Native Method)
  1536.         at org.apache.hadoop.hbase.regionserver.wal.SyncFuture.get(SyncFuture.java:167)
  1537.         - locked <0x0000000548a2ed88> (a org.apache.hadoop.hbase.regionserver.wal.SyncFuture)
  1538.         at org.apache.hadoop.hbase.regionserver.wal.FSHLog.blockOnSync(FSHLog.java:1472)
  1539.         at org.apache.hadoop.hbase.regionserver.wal.FSHLog.publishSyncThenBlockOnCompletion(FSHLog.java:1466)
  1540.         at org.apache.hadoop.hbase.regionserver.wal.FSHLog.sync(FSHLog.java:1600)
  1541.         at org.apache.hadoop.hbase.regionserver.HRegion.syncOrDefer(HRegion.java:7727)
  1542.         at org.apache.hadoop.hbase.regionserver.HRegion.doMiniBatchMutation(HRegion.java:3145)
  1543.         at org.apache.hadoop.hbase.regionserver.HRegion.batchMutate(HRegion.java:2793)
  1544.         at org.apache.hadoop.hbase.regionserver.HRegion.batchMutate(HRegion.java:2735)
  1545.         at org.apache.hadoop.hbase.regionserver.RSRpcServices.doBatchOp(RSRpcServices.java:692)
  1546.         at org.apache.hadoop.hbase.regionserver.RSRpcServices.doNonAtomicRegionMutation(RSRpcServices.java:654)
  1547.         at org.apache.hadoop.hbase.regionserver.RSRpcServices.multi(RSRpcServices.java:2029)
  1548.         at org.apache.hadoop.hbase.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:32213)
  1549.         at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:2112)
  1550.         at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:101)
  1551.         at org.apache.hadoop.hbase.ipc.RpcExecutor.consumerLoop(RpcExecutor.java:130)
  1552.         at org.apache.hadoop.hbase.ipc.RpcExecutor$1.run(RpcExecutor.java:107)
  1553.         at java.lang.Thread.run(Thread.java:745)
  1554.  
  1555.    Locked ownable synchronizers:
  1556.         - None
  1557.  
  1558. "B.defaultRpcServer.handler=25,queue=0,port=9104" #68 daemon prio=5 os_prio=0 tid=0x00007fbf7160f800 nid=0x73 in Object.wait() [0x00007fbf3e5de000]
  1559.    java.lang.Thread.State: TIMED_WAITING (on object monitor)
  1560.         at java.lang.Object.wait(Native Method)
  1561.         at org.apache.hadoop.hbase.regionserver.wal.SyncFuture.get(SyncFuture.java:167)
  1562.         - locked <0x0000000548b9b9c0> (a org.apache.hadoop.hbase.regionserver.wal.SyncFuture)
  1563.         at org.apache.hadoop.hbase.regionserver.wal.FSHLog.blockOnSync(FSHLog.java:1472)
  1564.         at org.apache.hadoop.hbase.regionserver.wal.FSHLog.publishSyncThenBlockOnCompletion(FSHLog.java:1466)
  1565.         at org.apache.hadoop.hbase.regionserver.wal.FSHLog.sync(FSHLog.java:1600)
  1566.         at org.apache.hadoop.hbase.regionserver.HRegion.syncOrDefer(HRegion.java:7727)
  1567.         at org.apache.hadoop.hbase.regionserver.HRegion.doMiniBatchMutation(HRegion.java:3145)
  1568.         at org.apache.hadoop.hbase.regionserver.HRegion.batchMutate(HRegion.java:2793)
  1569.         at org.apache.hadoop.hbase.regionserver.HRegion.batchMutate(HRegion.java:2735)
  1570.         at org.apache.hadoop.hbase.regionserver.RSRpcServices.doBatchOp(RSRpcServices.java:692)
  1571.         at org.apache.hadoop.hbase.regionserver.RSRpcServices.doNonAtomicRegionMutation(RSRpcServices.java:654)
  1572.         at org.apache.hadoop.hbase.regionserver.RSRpcServices.multi(RSRpcServices.java:2029)
  1573.         at org.apache.hadoop.hbase.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:32213)
  1574.         at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:2112)
  1575.         at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:101)
  1576.         at org.apache.hadoop.hbase.ipc.RpcExecutor.consumerLoop(RpcExecutor.java:130)
  1577.         at org.apache.hadoop.hbase.ipc.RpcExecutor$1.run(RpcExecutor.java:107)
  1578.         at java.lang.Thread.run(Thread.java:745)
  1579.  
  1580.    Locked ownable synchronizers:
  1581.         - None
  1582.  
  1583. "B.defaultRpcServer.handler=24,queue=4,port=9104" #67 daemon prio=5 os_prio=0 tid=0x00007fbf7160e000 nid=0x72 in Object.wait() [0x00007fbf3e6df000]
  1584.    java.lang.Thread.State: TIMED_WAITING (on object monitor)
  1585.         at java.lang.Object.wait(Native Method)
  1586.         at org.apache.hadoop.hbase.regionserver.wal.SyncFuture.get(SyncFuture.java:167)
  1587.         - locked <0x0000000548300ca0> (a org.apache.hadoop.hbase.regionserver.wal.SyncFuture)
  1588.         at org.apache.hadoop.hbase.regionserver.wal.FSHLog.blockOnSync(FSHLog.java:1472)
  1589.         at org.apache.hadoop.hbase.regionserver.wal.FSHLog.publishSyncThenBlockOnCompletion(FSHLog.java:1466)
  1590.         at org.apache.hadoop.hbase.regionserver.wal.FSHLog.sync(FSHLog.java:1600)
  1591.         at org.apache.hadoop.hbase.regionserver.HRegion.syncOrDefer(HRegion.java:7727)
  1592.         at org.apache.hadoop.hbase.regionserver.HRegion.doMiniBatchMutation(HRegion.java:3145)
  1593.         at org.apache.hadoop.hbase.regionserver.HRegion.batchMutate(HRegion.java:2793)
  1594.         at org.apache.hadoop.hbase.regionserver.HRegion.batchMutate(HRegion.java:2735)
  1595.         at org.apache.hadoop.hbase.regionserver.RSRpcServices.doBatchOp(RSRpcServices.java:692)
  1596.         at org.apache.hadoop.hbase.regionserver.RSRpcServices.doNonAtomicRegionMutation(RSRpcServices.java:654)
  1597.         at org.apache.hadoop.hbase.regionserver.RSRpcServices.multi(RSRpcServices.java:2029)
  1598.         at org.apache.hadoop.hbase.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:32213)
  1599.         at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:2112)
  1600.         at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:101)
  1601.         at org.apache.hadoop.hbase.ipc.RpcExecutor.consumerLoop(RpcExecutor.java:130)
  1602.         at org.apache.hadoop.hbase.ipc.RpcExecutor$1.run(RpcExecutor.java:107)
  1603.         at java.lang.Thread.run(Thread.java:745)
  1604.  
  1605.    Locked ownable synchronizers:
  1606.         - None
  1607.  
  1608. "B.defaultRpcServer.handler=23,queue=3,port=9104" #66 daemon prio=5 os_prio=0 tid=0x00007fbf7160c000 nid=0x71 in Object.wait() [0x00007fbf3e7e0000]
  1609.    java.lang.Thread.State: TIMED_WAITING (on object monitor)
  1610.         at java.lang.Object.wait(Native Method)
  1611.         at org.apache.hadoop.hbase.regionserver.wal.SyncFuture.get(SyncFuture.java:167)
  1612.         - locked <0x00000005489b00f0> (a org.apache.hadoop.hbase.regionserver.wal.SyncFuture)
  1613.         at org.apache.hadoop.hbase.regionserver.wal.FSHLog.blockOnSync(FSHLog.java:1472)
  1614.         at org.apache.hadoop.hbase.regionserver.wal.FSHLog.publishSyncThenBlockOnCompletion(FSHLog.java:1466)
  1615.         at org.apache.hadoop.hbase.regionserver.wal.FSHLog.sync(FSHLog.java:1600)
  1616.         at org.apache.hadoop.hbase.regionserver.HRegion.syncOrDefer(HRegion.java:7727)
  1617.         at org.apache.hadoop.hbase.regionserver.HRegion.doMiniBatchMutation(HRegion.java:3145)
  1618.         at org.apache.hadoop.hbase.regionserver.HRegion.batchMutate(HRegion.java:2793)
  1619.         at org.apache.hadoop.hbase.regionserver.HRegion.batchMutate(HRegion.java:2735)
  1620.         at org.apache.hadoop.hbase.regionserver.RSRpcServices.doBatchOp(RSRpcServices.java:692)
  1621.         at org.apache.hadoop.hbase.regionserver.RSRpcServices.doNonAtomicRegionMutation(RSRpcServices.java:654)
  1622.         at org.apache.hadoop.hbase.regionserver.RSRpcServices.multi(RSRpcServices.java:2029)
  1623.         at org.apache.hadoop.hbase.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:32213)
  1624.         at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:2112)
  1625.         at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:101)
  1626.         at org.apache.hadoop.hbase.ipc.RpcExecutor.consumerLoop(RpcExecutor.java:130)
  1627.         at org.apache.hadoop.hbase.ipc.RpcExecutor$1.run(RpcExecutor.java:107)
  1628.         at java.lang.Thread.run(Thread.java:745)
  1629.  
  1630.    Locked ownable synchronizers:
  1631.         - None
  1632.  
  1633. "B.defaultRpcServer.handler=22,queue=2,port=9104" #65 daemon prio=5 os_prio=0 tid=0x00007fbf7160a800 nid=0x70 in Object.wait() [0x00007fbf3e8e1000]
  1634.    java.lang.Thread.State: TIMED_WAITING (on object monitor)
  1635.         at java.lang.Object.wait(Native Method)
  1636.         at org.apache.hadoop.hbase.regionserver.wal.SyncFuture.get(SyncFuture.java:167)
  1637.         - locked <0x0000000548d42878> (a org.apache.hadoop.hbase.regionserver.wal.SyncFuture)
  1638.         at org.apache.hadoop.hbase.regionserver.wal.FSHLog.blockOnSync(FSHLog.java:1472)
  1639.         at org.apache.hadoop.hbase.regionserver.wal.FSHLog.publishSyncThenBlockOnCompletion(FSHLog.java:1466)
  1640.         at org.apache.hadoop.hbase.regionserver.wal.FSHLog.sync(FSHLog.java:1600)
  1641.         at org.apache.hadoop.hbase.regionserver.HRegion.syncOrDefer(HRegion.java:7727)
  1642.         at org.apache.hadoop.hbase.regionserver.HRegion.doMiniBatchMutation(HRegion.java:3145)
  1643.         at org.apache.hadoop.hbase.regionserver.HRegion.batchMutate(HRegion.java:2793)
  1644.         at org.apache.hadoop.hbase.regionserver.HRegion.batchMutate(HRegion.java:2735)
  1645.         at org.apache.hadoop.hbase.regionserver.RSRpcServices.doBatchOp(RSRpcServices.java:692)
  1646.         at org.apache.hadoop.hbase.regionserver.RSRpcServices.doNonAtomicRegionMutation(RSRpcServices.java:654)
  1647.         at org.apache.hadoop.hbase.regionserver.RSRpcServices.multi(RSRpcServices.java:2029)
  1648.         at org.apache.hadoop.hbase.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:32213)
  1649.         at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:2112)
  1650.         at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:101)
  1651.         at org.apache.hadoop.hbase.ipc.RpcExecutor.consumerLoop(RpcExecutor.java:130)
  1652.         at org.apache.hadoop.hbase.ipc.RpcExecutor$1.run(RpcExecutor.java:107)
  1653.         at java.lang.Thread.run(Thread.java:745)
  1654.  
  1655.    Locked ownable synchronizers:
  1656.         - None
  1657.  
  1658. "B.defaultRpcServer.handler=21,queue=1,port=9104" #64 daemon prio=5 os_prio=0 tid=0x00007fbf71608800 nid=0x6f in Object.wait() [0x00007fbf3e9e2000]
  1659.    java.lang.Thread.State: TIMED_WAITING (on object monitor)
  1660.         at java.lang.Object.wait(Native Method)
  1661.         at org.apache.hadoop.hbase.regionserver.wal.SyncFuture.get(SyncFuture.java:167)
  1662.         - locked <0x0000000548a2ece8> (a org.apache.hadoop.hbase.regionserver.wal.SyncFuture)
  1663.         at org.apache.hadoop.hbase.regionserver.wal.FSHLog.blockOnSync(FSHLog.java:1472)
  1664.         at org.apache.hadoop.hbase.regionserver.wal.FSHLog.publishSyncThenBlockOnCompletion(FSHLog.java:1466)
  1665.         at org.apache.hadoop.hbase.regionserver.wal.FSHLog.sync(FSHLog.java:1600)
  1666.         at org.apache.hadoop.hbase.regionserver.HRegion.syncOrDefer(HRegion.java:7727)
  1667.         at org.apache.hadoop.hbase.regionserver.HRegion.doMiniBatchMutation(HRegion.java:3145)
  1668.         at org.apache.hadoop.hbase.regionserver.HRegion.batchMutate(HRegion.java:2793)
  1669.         at org.apache.hadoop.hbase.regionserver.HRegion.batchMutate(HRegion.java:2735)
  1670.         at org.apache.hadoop.hbase.regionserver.RSRpcServices.doBatchOp(RSRpcServices.java:692)
  1671.         at org.apache.hadoop.hbase.regionserver.RSRpcServices.doNonAtomicRegionMutation(RSRpcServices.java:654)
  1672.         at org.apache.hadoop.hbase.regionserver.RSRpcServices.multi(RSRpcServices.java:2029)
  1673.         at org.apache.hadoop.hbase.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:32213)
  1674.         at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:2112)
  1675.         at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:101)
  1676.         at org.apache.hadoop.hbase.ipc.RpcExecutor.consumerLoop(RpcExecutor.java:130)
  1677.         at org.apache.hadoop.hbase.ipc.RpcExecutor$1.run(RpcExecutor.java:107)
  1678.         at java.lang.Thread.run(Thread.java:745)
  1679.  
  1680.    Locked ownable synchronizers:
  1681.         - None
  1682.  
  1683. "B.defaultRpcServer.handler=20,queue=0,port=9104" #63 daemon prio=5 os_prio=0 tid=0x00007fbf71607000 nid=0x6e in Object.wait() [0x00007fbf3eae3000]
  1684.    java.lang.Thread.State: TIMED_WAITING (on object monitor)
  1685.         at java.lang.Object.wait(Native Method)
  1686.         at org.apache.hadoop.hbase.regionserver.wal.SyncFuture.get(SyncFuture.java:167)
  1687.         - locked <0x000000054877e580> (a org.apache.hadoop.hbase.regionserver.wal.SyncFuture)
  1688.         at org.apache.hadoop.hbase.regionserver.wal.FSHLog.blockOnSync(FSHLog.java:1472)
  1689.         at org.apache.hadoop.hbase.regionserver.wal.FSHLog.publishSyncThenBlockOnCompletion(FSHLog.java:1466)
  1690.         at org.apache.hadoop.hbase.regionserver.wal.FSHLog.sync(FSHLog.java:1600)
  1691.         at org.apache.hadoop.hbase.regionserver.HRegion.syncOrDefer(HRegion.java:7727)
  1692.         at org.apache.hadoop.hbase.regionserver.HRegion.doMiniBatchMutation(HRegion.java:3145)
  1693.         at org.apache.hadoop.hbase.regionserver.HRegion.batchMutate(HRegion.java:2793)
  1694.         at org.apache.hadoop.hbase.regionserver.HRegion.batchMutate(HRegion.java:2735)
  1695.         at org.apache.hadoop.hbase.regionserver.RSRpcServices.doBatchOp(RSRpcServices.java:692)
  1696.         at org.apache.hadoop.hbase.regionserver.RSRpcServices.doNonAtomicRegionMutation(RSRpcServices.java:654)
  1697.         at org.apache.hadoop.hbase.regionserver.RSRpcServices.multi(RSRpcServices.java:2029)
  1698.         at org.apache.hadoop.hbase.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:32213)
  1699.         at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:2112)
  1700.         at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:101)
  1701.         at org.apache.hadoop.hbase.ipc.RpcExecutor.consumerLoop(RpcExecutor.java:130)
  1702.         at org.apache.hadoop.hbase.ipc.RpcExecutor$1.run(RpcExecutor.java:107)
  1703.         at java.lang.Thread.run(Thread.java:745)
  1704.  
  1705.    Locked ownable synchronizers:
  1706.         - None
  1707.  
  1708. "B.defaultRpcServer.handler=19,queue=4,port=9104" #62 daemon prio=5 os_prio=0 tid=0x00007fbf71605000 nid=0x6d in Object.wait() [0x00007fbf3ebe4000]
  1709.    java.lang.Thread.State: TIMED_WAITING (on object monitor)
  1710.         at java.lang.Object.wait(Native Method)
  1711.         at org.apache.hadoop.hbase.regionserver.wal.SyncFuture.get(SyncFuture.java:167)
  1712.         - locked <0x00000005482ffc88> (a org.apache.hadoop.hbase.regionserver.wal.SyncFuture)
  1713.         at org.apache.hadoop.hbase.regionserver.wal.FSHLog.blockOnSync(FSHLog.java:1472)
  1714.         at org.apache.hadoop.hbase.regionserver.wal.FSHLog.publishSyncThenBlockOnCompletion(FSHLog.java:1466)
  1715.         at org.apache.hadoop.hbase.regionserver.wal.FSHLog.sync(FSHLog.java:1600)
  1716.         at org.apache.hadoop.hbase.regionserver.HRegion.syncOrDefer(HRegion.java:7727)
  1717.         at org.apache.hadoop.hbase.regionserver.HRegion.doMiniBatchMutation(HRegion.java:3145)
  1718.         at org.apache.hadoop.hbase.regionserver.HRegion.batchMutate(HRegion.java:2793)
  1719.         at org.apache.hadoop.hbase.regionserver.HRegion.batchMutate(HRegion.java:2735)
  1720.         at org.apache.hadoop.hbase.regionserver.RSRpcServices.doBatchOp(RSRpcServices.java:692)
  1721.         at org.apache.hadoop.hbase.regionserver.RSRpcServices.doNonAtomicRegionMutation(RSRpcServices.java:654)
  1722.         at org.apache.hadoop.hbase.regionserver.RSRpcServices.multi(RSRpcServices.java:2029)
  1723.         at org.apache.hadoop.hbase.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:32213)
  1724.         at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:2112)
  1725.         at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:101)
  1726.         at org.apache.hadoop.hbase.ipc.RpcExecutor.consumerLoop(RpcExecutor.java:130)
  1727.         at org.apache.hadoop.hbase.ipc.RpcExecutor$1.run(RpcExecutor.java:107)
  1728.         at java.lang.Thread.run(Thread.java:745)
  1729.  
  1730.    Locked ownable synchronizers:
  1731.         - None
  1732.  
  1733. "B.defaultRpcServer.handler=18,queue=3,port=9104" #61 daemon prio=5 os_prio=0 tid=0x00007fbf715e5000 nid=0x6c in Object.wait() [0x00007fbf3ece5000]
  1734.    java.lang.Thread.State: TIMED_WAITING (on object monitor)
  1735.         at java.lang.Object.wait(Native Method)
  1736.         at org.apache.hadoop.hbase.regionserver.wal.SyncFuture.get(SyncFuture.java:167)
  1737.         - locked <0x0000000548d42b70> (a org.apache.hadoop.hbase.regionserver.wal.SyncFuture)
  1738.         at org.apache.hadoop.hbase.regionserver.wal.FSHLog.blockOnSync(FSHLog.java:1472)
  1739.         at org.apache.hadoop.hbase.regionserver.wal.FSHLog.publishSyncThenBlockOnCompletion(FSHLog.java:1466)
  1740.         at org.apache.hadoop.hbase.regionserver.wal.FSHLog.sync(FSHLog.java:1600)
  1741.         at org.apache.hadoop.hbase.regionserver.HRegion.syncOrDefer(HRegion.java:7727)
  1742.         at org.apache.hadoop.hbase.regionserver.HRegion.doMiniBatchMutation(HRegion.java:3145)
  1743.         at org.apache.hadoop.hbase.regionserver.HRegion.batchMutate(HRegion.java:2793)
  1744.         at org.apache.hadoop.hbase.regionserver.HRegion.batchMutate(HRegion.java:2735)
  1745.         at org.apache.hadoop.hbase.regionserver.RSRpcServices.doBatchOp(RSRpcServices.java:692)
  1746.         at org.apache.hadoop.hbase.regionserver.RSRpcServices.doNonAtomicRegionMutation(RSRpcServices.java:654)
  1747.         at org.apache.hadoop.hbase.regionserver.RSRpcServices.multi(RSRpcServices.java:2029)
  1748.         at org.apache.hadoop.hbase.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:32213)
  1749.         at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:2112)
  1750.         at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:101)
  1751.         at org.apache.hadoop.hbase.ipc.RpcExecutor.consumerLoop(RpcExecutor.java:130)
  1752.         at org.apache.hadoop.hbase.ipc.RpcExecutor$1.run(RpcExecutor.java:107)
  1753.         at java.lang.Thread.run(Thread.java:745)
  1754.  
  1755.    Locked ownable synchronizers:
  1756.         - None
  1757.  
  1758. "B.defaultRpcServer.handler=17,queue=2,port=9104" #60 daemon prio=5 os_prio=0 tid=0x00007fbf715e3000 nid=0x6b in Object.wait() [0x00007fbf3ede6000]
  1759.    java.lang.Thread.State: TIMED_WAITING (on object monitor)
  1760.         at java.lang.Object.wait(Native Method)
  1761.         at org.apache.hadoop.hbase.regionserver.wal.SyncFuture.get(SyncFuture.java:167)
  1762.         - locked <0x0000000548f7f2c0> (a org.apache.hadoop.hbase.regionserver.wal.SyncFuture)
  1763.         at org.apache.hadoop.hbase.regionserver.wal.FSHLog.blockOnSync(FSHLog.java:1472)
  1764.         at org.apache.hadoop.hbase.regionserver.wal.FSHLog.publishSyncThenBlockOnCompletion(FSHLog.java:1466)
  1765.         at org.apache.hadoop.hbase.regionserver.wal.FSHLog.sync(FSHLog.java:1600)
  1766.         at org.apache.hadoop.hbase.regionserver.HRegion.syncOrDefer(HRegion.java:7727)
  1767.         at org.apache.hadoop.hbase.regionserver.HRegion.doMiniBatchMutation(HRegion.java:3145)
  1768.         at org.apache.hadoop.hbase.regionserver.HRegion.batchMutate(HRegion.java:2793)
  1769.         at org.apache.hadoop.hbase.regionserver.HRegion.batchMutate(HRegion.java:2735)
  1770.         at org.apache.hadoop.hbase.regionserver.RSRpcServices.doBatchOp(RSRpcServices.java:692)
  1771.         at org.apache.hadoop.hbase.regionserver.RSRpcServices.doNonAtomicRegionMutation(RSRpcServices.java:654)
  1772.         at org.apache.hadoop.hbase.regionserver.RSRpcServices.multi(RSRpcServices.java:2029)
  1773.         at org.apache.hadoop.hbase.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:32213)
  1774.         at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:2112)
  1775.         at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:101)
  1776.         at org.apache.hadoop.hbase.ipc.RpcExecutor.consumerLoop(RpcExecutor.java:130)
  1777.         at org.apache.hadoop.hbase.ipc.RpcExecutor$1.run(RpcExecutor.java:107)
  1778.         at java.lang.Thread.run(Thread.java:745)
  1779.  
  1780.    Locked ownable synchronizers:
  1781.         - None
  1782.  
  1783. "B.defaultRpcServer.handler=16,queue=1,port=9104" #59 daemon prio=5 os_prio=0 tid=0x00007fbf715e1800 nid=0x6a in Object.wait() [0x00007fbf3eee7000]
  1784.    java.lang.Thread.State: TIMED_WAITING (on object monitor)
  1785.         at java.lang.Object.wait(Native Method)
  1786.         at org.apache.hadoop.hbase.regionserver.wal.SyncFuture.get(SyncFuture.java:167)
  1787.         - locked <0x0000000548a2ec48> (a org.apache.hadoop.hbase.regionserver.wal.SyncFuture)
  1788.         at org.apache.hadoop.hbase.regionserver.wal.FSHLog.blockOnSync(FSHLog.java:1472)
  1789.         at org.apache.hadoop.hbase.regionserver.wal.FSHLog.publishSyncThenBlockOnCompletion(FSHLog.java:1466)
  1790.         at org.apache.hadoop.hbase.regionserver.wal.FSHLog.sync(FSHLog.java:1600)
  1791.         at org.apache.hadoop.hbase.regionserver.HRegion.syncOrDefer(HRegion.java:7727)
  1792.         at org.apache.hadoop.hbase.regionserver.HRegion.doMiniBatchMutation(HRegion.java:3145)
  1793.         at org.apache.hadoop.hbase.regionserver.HRegion.batchMutate(HRegion.java:2793)
  1794.         at org.apache.hadoop.hbase.regionserver.HRegion.batchMutate(HRegion.java:2735)
  1795.         at org.apache.hadoop.hbase.regionserver.RSRpcServices.doBatchOp(RSRpcServices.java:692)
  1796.         at org.apache.hadoop.hbase.regionserver.RSRpcServices.doNonAtomicRegionMutation(RSRpcServices.java:654)
  1797.         at org.apache.hadoop.hbase.regionserver.RSRpcServices.multi(RSRpcServices.java:2029)
  1798.         at org.apache.hadoop.hbase.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:32213)
  1799.         at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:2112)
  1800.         at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:101)
  1801.         at org.apache.hadoop.hbase.ipc.RpcExecutor.consumerLoop(RpcExecutor.java:130)
  1802.         at org.apache.hadoop.hbase.ipc.RpcExecutor$1.run(RpcExecutor.java:107)
  1803.         at java.lang.Thread.run(Thread.java:745)
  1804.  
  1805.    Locked ownable synchronizers:
  1806.         - None
  1807.  
  1808. "B.defaultRpcServer.handler=15,queue=0,port=9104" #58 daemon prio=5 os_prio=0 tid=0x00007fbf715df800 nid=0x69 in Object.wait() [0x00007fbf3efe8000]
  1809.    java.lang.Thread.State: TIMED_WAITING (on object monitor)
  1810.         at java.lang.Object.wait(Native Method)
  1811.         at org.apache.hadoop.hbase.regionserver.wal.SyncFuture.get(SyncFuture.java:167)
  1812.         - locked <0x0000000548f6b060> (a org.apache.hadoop.hbase.regionserver.wal.SyncFuture)
  1813.         at org.apache.hadoop.hbase.regionserver.wal.FSHLog.blockOnSync(FSHLog.java:1472)
  1814.         at org.apache.hadoop.hbase.regionserver.wal.FSHLog.publishSyncThenBlockOnCompletion(FSHLog.java:1466)
  1815.         at org.apache.hadoop.hbase.regionserver.wal.FSHLog.sync(FSHLog.java:1600)
  1816.         at org.apache.hadoop.hbase.regionserver.HRegion.syncOrDefer(HRegion.java:7727)
  1817.         at org.apache.hadoop.hbase.regionserver.HRegion.doMiniBatchMutation(HRegion.java:3145)
  1818.         at org.apache.hadoop.hbase.regionserver.HRegion.batchMutate(HRegion.java:2793)
  1819.         at org.apache.hadoop.hbase.regionserver.HRegion.batchMutate(HRegion.java:2735)
  1820.         at org.apache.hadoop.hbase.regionserver.RSRpcServices.doBatchOp(RSRpcServices.java:692)
  1821.         at org.apache.hadoop.hbase.regionserver.RSRpcServices.doNonAtomicRegionMutation(RSRpcServices.java:654)
  1822.         at org.apache.hadoop.hbase.regionserver.RSRpcServices.multi(RSRpcServices.java:2029)
  1823.         at org.apache.hadoop.hbase.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:32213)
  1824.         at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:2112)
  1825.         at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:101)
  1826.         at org.apache.hadoop.hbase.ipc.RpcExecutor.consumerLoop(RpcExecutor.java:130)
  1827.         at org.apache.hadoop.hbase.ipc.RpcExecutor$1.run(RpcExecutor.java:107)
  1828.         at java.lang.Thread.run(Thread.java:745)
  1829.  
  1830.    Locked ownable synchronizers:
  1831.         - None
  1832.  
  1833. "B.defaultRpcServer.handler=14,queue=4,port=9104" #57 daemon prio=5 os_prio=0 tid=0x00007fbf715de000 nid=0x68 in Object.wait() [0x00007fbf3f0e9000]
  1834.    java.lang.Thread.State: TIMED_WAITING (on object monitor)
  1835.         at java.lang.Object.wait(Native Method)
  1836.         at org.apache.hadoop.hbase.regionserver.wal.SyncFuture.get(SyncFuture.java:167)
  1837.         - locked <0x0000000548d429b8> (a org.apache.hadoop.hbase.regionserver.wal.SyncFuture)
  1838.         at org.apache.hadoop.hbase.regionserver.wal.FSHLog.blockOnSync(FSHLog.java:1472)
  1839.         at org.apache.hadoop.hbase.regionserver.wal.FSHLog.publishSyncThenBlockOnCompletion(FSHLog.java:1466)
  1840.         at org.apache.hadoop.hbase.regionserver.wal.FSHLog.sync(FSHLog.java:1600)
  1841.         at org.apache.hadoop.hbase.regionserver.HRegion.syncOrDefer(HRegion.java:7727)
  1842.         at org.apache.hadoop.hbase.regionserver.HRegion.doMiniBatchMutation(HRegion.java:3145)
  1843.         at org.apache.hadoop.hbase.regionserver.HRegion.batchMutate(HRegion.java:2793)
  1844.         at org.apache.hadoop.hbase.regionserver.HRegion.batchMutate(HRegion.java:2735)
  1845.         at org.apache.hadoop.hbase.regionserver.RSRpcServices.doBatchOp(RSRpcServices.java:692)
  1846.         at org.apache.hadoop.hbase.regionserver.RSRpcServices.doNonAtomicRegionMutation(RSRpcServices.java:654)
  1847.         at org.apache.hadoop.hbase.regionserver.RSRpcServices.multi(RSRpcServices.java:2029)
  1848.         at org.apache.hadoop.hbase.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:32213)
  1849.         at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:2112)
  1850.         at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:101)
  1851.         at org.apache.hadoop.hbase.ipc.RpcExecutor.consumerLoop(RpcExecutor.java:130)
  1852.         at org.apache.hadoop.hbase.ipc.RpcExecutor$1.run(RpcExecutor.java:107)
  1853.         at java.lang.Thread.run(Thread.java:745)
  1854.  
  1855.    Locked ownable synchronizers:
  1856.         - None
  1857.  
  1858. "B.defaultRpcServer.handler=13,queue=3,port=9104" #56 daemon prio=5 os_prio=0 tid=0x00007fbf715dc000 nid=0x67 in Object.wait() [0x00007fbf3f1ea000]
  1859.    java.lang.Thread.State: TIMED_WAITING (on object monitor)
  1860.         at java.lang.Object.wait(Native Method)
  1861.         at org.apache.hadoop.hbase.regionserver.wal.SyncFuture.get(SyncFuture.java:167)
  1862.         - locked <0x0000000548d42cd8> (a org.apache.hadoop.hbase.regionserver.wal.SyncFuture)
  1863.         at org.apache.hadoop.hbase.regionserver.wal.FSHLog.blockOnSync(FSHLog.java:1472)
  1864.         at org.apache.hadoop.hbase.regionserver.wal.FSHLog.publishSyncThenBlockOnCompletion(FSHLog.java:1466)
  1865.         at org.apache.hadoop.hbase.regionserver.wal.FSHLog.sync(FSHLog.java:1600)
  1866.         at org.apache.hadoop.hbase.regionserver.HRegion.syncOrDefer(HRegion.java:7727)
  1867.         at org.apache.hadoop.hbase.regionserver.HRegion.doMiniBatchMutation(HRegion.java:3145)
  1868.         at org.apache.hadoop.hbase.regionserver.HRegion.batchMutate(HRegion.java:2793)
  1869.         at org.apache.hadoop.hbase.regionserver.HRegion.batchMutate(HRegion.java:2735)
  1870.         at org.apache.hadoop.hbase.regionserver.RSRpcServices.doBatchOp(RSRpcServices.java:692)
  1871.         at org.apache.hadoop.hbase.regionserver.RSRpcServices.doNonAtomicRegionMutation(RSRpcServices.java:654)
  1872.         at org.apache.hadoop.hbase.regionserver.RSRpcServices.multi(RSRpcServices.java:2029)
  1873.         at org.apache.hadoop.hbase.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:32213)
  1874.         at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:2112)
  1875.         at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:101)
  1876.         at org.apache.hadoop.hbase.ipc.RpcExecutor.consumerLoop(RpcExecutor.java:130)
  1877.         at org.apache.hadoop.hbase.ipc.RpcExecutor$1.run(RpcExecutor.java:107)
  1878.         at java.lang.Thread.run(Thread.java:745)
  1879.  
  1880.    Locked ownable synchronizers:
  1881.         - None
  1882.  
  1883. "B.defaultRpcServer.handler=12,queue=2,port=9104" #55 daemon prio=5 os_prio=0 tid=0x00007fbf715da800 nid=0x66 in Object.wait() [0x00007fbf3f2eb000]
  1884.    java.lang.Thread.State: TIMED_WAITING (on object monitor)
  1885.         at java.lang.Object.wait(Native Method)
  1886.         at org.apache.hadoop.hbase.regionserver.wal.SyncFuture.get(SyncFuture.java:167)
  1887.         - locked <0x0000000548a2ed10> (a org.apache.hadoop.hbase.regionserver.wal.SyncFuture)
  1888.         at org.apache.hadoop.hbase.regionserver.wal.FSHLog.blockOnSync(FSHLog.java:1472)
  1889.         at org.apache.hadoop.hbase.regionserver.wal.FSHLog.publishSyncThenBlockOnCompletion(FSHLog.java:1466)
  1890.         at org.apache.hadoop.hbase.regionserver.wal.FSHLog.sync(FSHLog.java:1600)
  1891.         at org.apache.hadoop.hbase.regionserver.HRegion.syncOrDefer(HRegion.java:7727)
  1892.         at org.apache.hadoop.hbase.regionserver.HRegion.doMiniBatchMutation(HRegion.java:3145)
  1893.         at org.apache.hadoop.hbase.regionserver.HRegion.batchMutate(HRegion.java:2793)
  1894.         at org.apache.hadoop.hbase.regionserver.HRegion.batchMutate(HRegion.java:2735)
  1895.         at org.apache.hadoop.hbase.regionserver.RSRpcServices.doBatchOp(RSRpcServices.java:692)
  1896.         at org.apache.hadoop.hbase.regionserver.RSRpcServices.doNonAtomicRegionMutation(RSRpcServices.java:654)
  1897.         at org.apache.hadoop.hbase.regionserver.RSRpcServices.multi(RSRpcServices.java:2029)
  1898.         at org.apache.hadoop.hbase.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:32213)
  1899.         at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:2112)
  1900.         at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:101)
  1901.         at org.apache.hadoop.hbase.ipc.RpcExecutor.consumerLoop(RpcExecutor.java:130)
  1902.         at org.apache.hadoop.hbase.ipc.RpcExecutor$1.run(RpcExecutor.java:107)
  1903.         at java.lang.Thread.run(Thread.java:745)
  1904.  
  1905.    Locked ownable synchronizers:
  1906.         - None
  1907.  
  1908. "B.defaultRpcServer.handler=11,queue=1,port=9104" #54 daemon prio=5 os_prio=0 tid=0x00007fbf715d8800 nid=0x65 in Object.wait() [0x00007fbf3f3ec000]
  1909.    java.lang.Thread.State: TIMED_WAITING (on object monitor)
  1910.         at java.lang.Object.wait(Native Method)
  1911.         at org.apache.hadoop.hbase.regionserver.wal.SyncFuture.get(SyncFuture.java:167)
  1912.         - locked <0x0000000548a2ed60> (a org.apache.hadoop.hbase.regionserver.wal.SyncFuture)
  1913.         at org.apache.hadoop.hbase.regionserver.wal.FSHLog.blockOnSync(FSHLog.java:1472)
  1914.         at org.apache.hadoop.hbase.regionserver.wal.FSHLog.publishSyncThenBlockOnCompletion(FSHLog.java:1466)
  1915.         at org.apache.hadoop.hbase.regionserver.wal.FSHLog.sync(FSHLog.java:1600)
  1916.         at org.apache.hadoop.hbase.regionserver.HRegion.syncOrDefer(HRegion.java:7727)
  1917.         at org.apache.hadoop.hbase.regionserver.HRegion.doMiniBatchMutation(HRegion.java:3145)
  1918.         at org.apache.hadoop.hbase.regionserver.HRegion.batchMutate(HRegion.java:2793)
  1919.         at org.apache.hadoop.hbase.regionserver.HRegion.batchMutate(HRegion.java:2735)
  1920.         at org.apache.hadoop.hbase.regionserver.RSRpcServices.doBatchOp(RSRpcServices.java:692)
  1921.         at org.apache.hadoop.hbase.regionserver.RSRpcServices.doNonAtomicRegionMutation(RSRpcServices.java:654)
  1922.         at org.apache.hadoop.hbase.regionserver.RSRpcServices.multi(RSRpcServices.java:2029)
  1923.         at org.apache.hadoop.hbase.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:32213)
  1924.         at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:2112)
  1925.         at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:101)
  1926.         at org.apache.hadoop.hbase.ipc.RpcExecutor.consumerLoop(RpcExecutor.java:130)
  1927.         at org.apache.hadoop.hbase.ipc.RpcExecutor$1.run(RpcExecutor.java:107)
  1928.         at java.lang.Thread.run(Thread.java:745)
  1929.  
  1930.    Locked ownable synchronizers:
  1931.         - None
  1932.  
  1933. "B.defaultRpcServer.handler=10,queue=0,port=9104" #53 daemon prio=5 os_prio=0 tid=0x00007fbf715d7000 nid=0x64 in Object.wait() [0x00007fbf3f4ed000]
  1934.    java.lang.Thread.State: TIMED_WAITING (on object monitor)
  1935.         at java.lang.Object.wait(Native Method)
  1936.         at org.apache.hadoop.hbase.regionserver.wal.SyncFuture.get(SyncFuture.java:167)
  1937.         - locked <0x0000000548b9b998> (a org.apache.hadoop.hbase.regionserver.wal.SyncFuture)
  1938.         at org.apache.hadoop.hbase.regionserver.wal.FSHLog.blockOnSync(FSHLog.java:1472)
  1939.         at org.apache.hadoop.hbase.regionserver.wal.FSHLog.publishSyncThenBlockOnCompletion(FSHLog.java:1466)
  1940.         at org.apache.hadoop.hbase.regionserver.wal.FSHLog.sync(FSHLog.java:1600)
  1941.         at org.apache.hadoop.hbase.regionserver.HRegion.syncOrDefer(HRegion.java:7727)
  1942.         at org.apache.hadoop.hbase.regionserver.HRegion.doMiniBatchMutation(HRegion.java:3145)
  1943.         at org.apache.hadoop.hbase.regionserver.HRegion.batchMutate(HRegion.java:2793)
  1944.         at org.apache.hadoop.hbase.regionserver.HRegion.batchMutate(HRegion.java:2735)
  1945.         at org.apache.hadoop.hbase.regionserver.RSRpcServices.doBatchOp(RSRpcServices.java:692)
  1946.         at org.apache.hadoop.hbase.regionserver.RSRpcServices.doNonAtomicRegionMutation(RSRpcServices.java:654)
  1947.         at org.apache.hadoop.hbase.regionserver.RSRpcServices.multi(RSRpcServices.java:2029)
  1948.         at org.apache.hadoop.hbase.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:32213)
  1949.         at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:2112)
  1950.         at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:101)
  1951.         at org.apache.hadoop.hbase.ipc.RpcExecutor.consumerLoop(RpcExecutor.java:130)
  1952.         at org.apache.hadoop.hbase.ipc.RpcExecutor$1.run(RpcExecutor.java:107)
  1953.         at java.lang.Thread.run(Thread.java:745)
  1954.  
  1955.    Locked ownable synchronizers:
  1956.         - None
  1957.  
  1958. "B.defaultRpcServer.handler=9,queue=4,port=9104" #52 daemon prio=5 os_prio=0 tid=0x00007fbf7157d000 nid=0x63 in Object.wait() [0x00007fbf3f5ee000]
  1959.    java.lang.Thread.State: TIMED_WAITING (on object monitor)
  1960.         at java.lang.Object.wait(Native Method)
  1961.         at org.apache.hadoop.hbase.regionserver.wal.SyncFuture.get(SyncFuture.java:167)
  1962.         - locked <0x0000000548a2ecc0> (a org.apache.hadoop.hbase.regionserver.wal.SyncFuture)
  1963.         at org.apache.hadoop.hbase.regionserver.wal.FSHLog.blockOnSync(FSHLog.java:1472)
  1964.         at org.apache.hadoop.hbase.regionserver.wal.FSHLog.publishSyncThenBlockOnCompletion(FSHLog.java:1466)
  1965.         at org.apache.hadoop.hbase.regionserver.wal.FSHLog.sync(FSHLog.java:1600)
  1966.         at org.apache.hadoop.hbase.regionserver.HRegion.syncOrDefer(HRegion.java:7727)
  1967.         at org.apache.hadoop.hbase.regionserver.HRegion.doMiniBatchMutation(HRegion.java:3145)
  1968.         at org.apache.hadoop.hbase.regionserver.HRegion.batchMutate(HRegion.java:2793)
  1969.         at org.apache.hadoop.hbase.regionserver.HRegion.batchMutate(HRegion.java:2735)
  1970.         at org.apache.hadoop.hbase.regionserver.RSRpcServices.doBatchOp(RSRpcServices.java:692)
  1971.         at org.apache.hadoop.hbase.regionserver.RSRpcServices.doNonAtomicRegionMutation(RSRpcServices.java:654)
  1972.         at org.apache.hadoop.hbase.regionserver.RSRpcServices.multi(RSRpcServices.java:2029)
  1973.         at org.apache.hadoop.hbase.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:32213)
  1974.         at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:2112)
  1975.         at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:101)
  1976.         at org.apache.hadoop.hbase.ipc.RpcExecutor.consumerLoop(RpcExecutor.java:130)
  1977.         at org.apache.hadoop.hbase.ipc.RpcExecutor$1.run(RpcExecutor.java:107)
  1978.         at java.lang.Thread.run(Thread.java:745)
  1979.  
  1980.    Locked ownable synchronizers:
  1981.         - None
  1982.  
  1983. "B.defaultRpcServer.handler=8,queue=3,port=9104" #51 daemon prio=5 os_prio=0 tid=0x00007fbf7157b000 nid=0x62 in Object.wait() [0x00007fbf3f6ef000]
  1984.    java.lang.Thread.State: TIMED_WAITING (on object monitor)
  1985.         at java.lang.Object.wait(Native Method)
  1986.         at org.apache.hadoop.hbase.regionserver.wal.SyncFuture.get(SyncFuture.java:167)
  1987.         - locked <0x0000000548a2edd8> (a org.apache.hadoop.hbase.regionserver.wal.SyncFuture)
  1988.         at org.apache.hadoop.hbase.regionserver.wal.FSHLog.blockOnSync(FSHLog.java:1472)
  1989.         at org.apache.hadoop.hbase.regionserver.wal.FSHLog.publishSyncThenBlockOnCompletion(FSHLog.java:1466)
  1990.         at org.apache.hadoop.hbase.regionserver.wal.FSHLog.sync(FSHLog.java:1600)
  1991.         at org.apache.hadoop.hbase.regionserver.HRegion.syncOrDefer(HRegion.java:7727)
  1992.         at org.apache.hadoop.hbase.regionserver.HRegion.doMiniBatchMutation(HRegion.java:3145)
  1993.         at org.apache.hadoop.hbase.regionserver.HRegion.batchMutate(HRegion.java:2793)
  1994.         at org.apache.hadoop.hbase.regionserver.HRegion.batchMutate(HRegion.java:2735)
  1995.         at org.apache.hadoop.hbase.regionserver.RSRpcServices.doBatchOp(RSRpcServices.java:692)
  1996.         at org.apache.hadoop.hbase.regionserver.RSRpcServices.doNonAtomicRegionMutation(RSRpcServices.java:654)
  1997.         at org.apache.hadoop.hbase.regionserver.RSRpcServices.multi(RSRpcServices.java:2029)
  1998.         at org.apache.hadoop.hbase.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:32213)
  1999.         at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:2112)
  2000.         at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:101)
  2001.         at org.apache.hadoop.hbase.ipc.RpcExecutor.consumerLoop(RpcExecutor.java:130)
  2002.         at org.apache.hadoop.hbase.ipc.RpcExecutor$1.run(RpcExecutor.java:107)
  2003.         at java.lang.Thread.run(Thread.java:745)
  2004.  
  2005.    Locked ownable synchronizers:
  2006.         - None
  2007.  
  2008. "B.defaultRpcServer.handler=7,queue=2,port=9104" #50 daemon prio=5 os_prio=0 tid=0x00007fbf71579000 nid=0x61 in Object.wait() [0x00007fbf3f7f0000]
  2009.    java.lang.Thread.State: TIMED_WAITING (on object monitor)
  2010.         at java.lang.Object.wait(Native Method)
  2011.         at org.apache.hadoop.hbase.regionserver.wal.SyncFuture.get(SyncFuture.java:167)
  2012.         - locked <0x0000000548a2edb0> (a org.apache.hadoop.hbase.regionserver.wal.SyncFuture)
  2013.         at org.apache.hadoop.hbase.regionserver.wal.FSHLog.blockOnSync(FSHLog.java:1472)
  2014.         at org.apache.hadoop.hbase.regionserver.wal.FSHLog.publishSyncThenBlockOnCompletion(FSHLog.java:1466)
  2015.         at org.apache.hadoop.hbase.regionserver.wal.FSHLog.sync(FSHLog.java:1600)
  2016.         at org.apache.hadoop.hbase.regionserver.HRegion.syncOrDefer(HRegion.java:7727)
  2017.         at org.apache.hadoop.hbase.regionserver.HRegion.doMiniBatchMutation(HRegion.java:3145)
  2018.         at org.apache.hadoop.hbase.regionserver.HRegion.batchMutate(HRegion.java:2793)
  2019.         at org.apache.hadoop.hbase.regionserver.HRegion.batchMutate(HRegion.java:2735)
  2020.         at org.apache.hadoop.hbase.regionserver.RSRpcServices.doBatchOp(RSRpcServices.java:692)
  2021.         at org.apache.hadoop.hbase.regionserver.RSRpcServices.doNonAtomicRegionMutation(RSRpcServices.java:654)
  2022.         at org.apache.hadoop.hbase.regionserver.RSRpcServices.multi(RSRpcServices.java:2029)
  2023.         at org.apache.hadoop.hbase.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:32213)
  2024.         at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:2112)
  2025.         at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:101)
  2026.         at org.apache.hadoop.hbase.ipc.RpcExecutor.consumerLoop(RpcExecutor.java:130)
  2027.         at org.apache.hadoop.hbase.ipc.RpcExecutor$1.run(RpcExecutor.java:107)
  2028.         at java.lang.Thread.run(Thread.java:745)
  2029.  
  2030.    Locked ownable synchronizers:
  2031.         - None
  2032.  
  2033. "B.defaultRpcServer.handler=6,queue=1,port=9104" #49 daemon prio=5 os_prio=0 tid=0x00007fbf71577800 nid=0x60 in Object.wait() [0x00007fbf3f8f1000]
  2034.    java.lang.Thread.State: TIMED_WAITING (on object monitor)
  2035.         at java.lang.Object.wait(Native Method)
  2036.         at org.apache.hadoop.hbase.regionserver.wal.SyncFuture.get(SyncFuture.java:167)
  2037.         - locked <0x0000000548a2ed38> (a org.apache.hadoop.hbase.regionserver.wal.SyncFuture)
  2038.         at org.apache.hadoop.hbase.regionserver.wal.FSHLog.blockOnSync(FSHLog.java:1472)
  2039.         at org.apache.hadoop.hbase.regionserver.wal.FSHLog.publishSyncThenBlockOnCompletion(FSHLog.java:1466)
  2040.         at org.apache.hadoop.hbase.regionserver.wal.FSHLog.sync(FSHLog.java:1600)
  2041.         at org.apache.hadoop.hbase.regionserver.HRegion.syncOrDefer(HRegion.java:7727)
  2042.         at org.apache.hadoop.hbase.regionserver.HRegion.doMiniBatchMutation(HRegion.java:3145)
  2043.         at org.apache.hadoop.hbase.regionserver.HRegion.batchMutate(HRegion.java:2793)
  2044.         at org.apache.hadoop.hbase.regionserver.HRegion.batchMutate(HRegion.java:2735)
  2045.         at org.apache.hadoop.hbase.regionserver.RSRpcServices.doBatchOp(RSRpcServices.java:692)
  2046.         at org.apache.hadoop.hbase.regionserver.RSRpcServices.doNonAtomicRegionMutation(RSRpcServices.java:654)
  2047.         at org.apache.hadoop.hbase.regionserver.RSRpcServices.multi(RSRpcServices.java:2029)
  2048.         at org.apache.hadoop.hbase.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:32213)
  2049.         at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:2112)
  2050.         at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:101)
  2051.         at org.apache.hadoop.hbase.ipc.RpcExecutor.consumerLoop(RpcExecutor.java:130)
  2052.         at org.apache.hadoop.hbase.ipc.RpcExecutor$1.run(RpcExecutor.java:107)
  2053.         at java.lang.Thread.run(Thread.java:745)
  2054.  
  2055.    Locked ownable synchronizers:
  2056.         - None
  2057.  
  2058. "B.defaultRpcServer.handler=5,queue=0,port=9104" #48 daemon prio=5 os_prio=0 tid=0x00007fbf71576000 nid=0x5f in Object.wait() [0x00007fbf3f9f2000]
  2059.    java.lang.Thread.State: TIMED_WAITING (on object monitor)
  2060.         at java.lang.Object.wait(Native Method)
  2061.         at org.apache.hadoop.hbase.regionserver.wal.SyncFuture.get(SyncFuture.java:167)
  2062.         - locked <0x0000000548f8b638> (a org.apache.hadoop.hbase.regionserver.wal.SyncFuture)
  2063.         at org.apache.hadoop.hbase.regionserver.wal.FSHLog.blockOnSync(FSHLog.java:1472)
  2064.         at org.apache.hadoop.hbase.regionserver.wal.FSHLog.publishSyncThenBlockOnCompletion(FSHLog.java:1466)
  2065.         at org.apache.hadoop.hbase.regionserver.wal.FSHLog.sync(FSHLog.java:1600)
  2066.         at org.apache.hadoop.hbase.regionserver.HRegion.syncOrDefer(HRegion.java:7727)
  2067.         at org.apache.hadoop.hbase.regionserver.HRegion.doMiniBatchMutation(HRegion.java:3145)
  2068.         at org.apache.hadoop.hbase.regionserver.HRegion.batchMutate(HRegion.java:2793)
  2069.         at org.apache.hadoop.hbase.regionserver.HRegion.batchMutate(HRegion.java:2735)
  2070.         at org.apache.hadoop.hbase.regionserver.RSRpcServices.doBatchOp(RSRpcServices.java:692)
  2071.         at org.apache.hadoop.hbase.regionserver.RSRpcServices.doNonAtomicRegionMutation(RSRpcServices.java:654)
  2072.         at org.apache.hadoop.hbase.regionserver.RSRpcServices.multi(RSRpcServices.java:2029)
  2073.         at org.apache.hadoop.hbase.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:32213)
  2074.         at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:2112)
  2075.         at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:101)
  2076.         at org.apache.hadoop.hbase.ipc.RpcExecutor.consumerLoop(RpcExecutor.java:130)
  2077.         at org.apache.hadoop.hbase.ipc.RpcExecutor$1.run(RpcExecutor.java:107)
  2078.         at java.lang.Thread.run(Thread.java:745)
  2079.  
  2080.    Locked ownable synchronizers:
  2081.         - None
  2082.  
  2083. "B.defaultRpcServer.handler=4,queue=4,port=9104" #47 daemon prio=5 os_prio=0 tid=0x00007fbf715ff000 nid=0x5e in Object.wait() [0x00007fbf3faf3000]
  2084.    java.lang.Thread.State: TIMED_WAITING (on object monitor)
  2085.         at java.lang.Object.wait(Native Method)
  2086.         at org.apache.hadoop.hbase.regionserver.wal.SyncFuture.get(SyncFuture.java:167)
  2087.         - locked <0x0000000548a2ee00> (a org.apache.hadoop.hbase.regionserver.wal.SyncFuture)
  2088.         at org.apache.hadoop.hbase.regionserver.wal.FSHLog.blockOnSync(FSHLog.java:1472)
  2089.         at org.apache.hadoop.hbase.regionserver.wal.FSHLog.publishSyncThenBlockOnCompletion(FSHLog.java:1466)
  2090.         at org.apache.hadoop.hbase.regionserver.wal.FSHLog.sync(FSHLog.java:1600)
  2091.         at org.apache.hadoop.hbase.regionserver.HRegion.syncOrDefer(HRegion.java:7727)
  2092.         at org.apache.hadoop.hbase.regionserver.HRegion.doMiniBatchMutation(HRegion.java:3145)
  2093.         at org.apache.hadoop.hbase.regionserver.HRegion.batchMutate(HRegion.java:2793)
  2094.         at org.apache.hadoop.hbase.regionserver.HRegion.batchMutate(HRegion.java:2735)
  2095.         at org.apache.hadoop.hbase.regionserver.RSRpcServices.doBatchOp(RSRpcServices.java:692)
  2096.         at org.apache.hadoop.hbase.regionserver.RSRpcServices.doNonAtomicRegionMutation(RSRpcServices.java:654)
  2097.         at org.apache.hadoop.hbase.regionserver.RSRpcServices.multi(RSRpcServices.java:2029)
  2098.         at org.apache.hadoop.hbase.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:32213)
  2099.         at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:2112)
  2100.         at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:101)
  2101.         at org.apache.hadoop.hbase.ipc.RpcExecutor.consumerLoop(RpcExecutor.java:130)
  2102.         at org.apache.hadoop.hbase.ipc.RpcExecutor$1.run(RpcExecutor.java:107)
  2103.         at java.lang.Thread.run(Thread.java:745)
  2104.  
  2105.    Locked ownable synchronizers:
  2106.         - None
  2107.  
  2108. "B.defaultRpcServer.handler=3,queue=3,port=9104" #46 daemon prio=5 os_prio=0 tid=0x00007fbf715fd800 nid=0x5d in Object.wait() [0x00007fbf3fbf4000]
  2109.    java.lang.Thread.State: TIMED_WAITING (on object monitor)
  2110.         at java.lang.Object.wait(Native Method)
  2111.         at org.apache.hadoop.hbase.regionserver.wal.SyncFuture.get(SyncFuture.java:167)
  2112.         - locked <0x0000000548a2ec70> (a org.apache.hadoop.hbase.regionserver.wal.SyncFuture)
  2113.         at org.apache.hadoop.hbase.regionserver.wal.FSHLog.blockOnSync(FSHLog.java:1472)
  2114.         at org.apache.hadoop.hbase.regionserver.wal.FSHLog.publishSyncThenBlockOnCompletion(FSHLog.java:1466)
  2115.         at org.apache.hadoop.hbase.regionserver.wal.FSHLog.sync(FSHLog.java:1600)
  2116.         at org.apache.hadoop.hbase.regionserver.HRegion.syncOrDefer(HRegion.java:7727)
  2117.         at org.apache.hadoop.hbase.regionserver.HRegion.doMiniBatchMutation(HRegion.java:3145)
  2118.         at org.apache.hadoop.hbase.regionserver.HRegion.batchMutate(HRegion.java:2793)
  2119.         at org.apache.hadoop.hbase.regionserver.HRegion.batchMutate(HRegion.java:2735)
  2120.         at org.apache.hadoop.hbase.regionserver.RSRpcServices.doBatchOp(RSRpcServices.java:692)
  2121.         at org.apache.hadoop.hbase.regionserver.RSRpcServices.doNonAtomicRegionMutation(RSRpcServices.java:654)
  2122.         at org.apache.hadoop.hbase.regionserver.RSRpcServices.multi(RSRpcServices.java:2029)
  2123.         at org.apache.hadoop.hbase.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:32213)
  2124.         at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:2112)
  2125.         at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:101)
  2126.         at org.apache.hadoop.hbase.ipc.RpcExecutor.consumerLoop(RpcExecutor.java:130)
  2127.         at org.apache.hadoop.hbase.ipc.RpcExecutor$1.run(RpcExecutor.java:107)
  2128.         at java.lang.Thread.run(Thread.java:745)
  2129.  
  2130.    Locked ownable synchronizers:
  2131.         - None
  2132.  
  2133. "B.defaultRpcServer.handler=2,queue=2,port=9104" #45 daemon prio=5 os_prio=0 tid=0x00007fbf715fb800 nid=0x5c in Object.wait() [0x00007fbf3fcf5000]
  2134.    java.lang.Thread.State: TIMED_WAITING (on object monitor)
  2135.         at java.lang.Object.wait(Native Method)
  2136.         at org.apache.hadoop.hbase.regionserver.wal.SyncFuture.get(SyncFuture.java:167)
  2137.         - locked <0x00000005487fcc28> (a org.apache.hadoop.hbase.regionserver.wal.SyncFuture)
  2138.         at org.apache.hadoop.hbase.regionserver.wal.FSHLog.blockOnSync(FSHLog.java:1472)
  2139.         at org.apache.hadoop.hbase.regionserver.wal.FSHLog.publishSyncThenBlockOnCompletion(FSHLog.java:1466)
  2140.         at org.apache.hadoop.hbase.regionserver.wal.FSHLog.sync(FSHLog.java:1600)
  2141.         at org.apache.hadoop.hbase.regionserver.HRegion.syncOrDefer(HRegion.java:7727)
  2142.         at org.apache.hadoop.hbase.regionserver.HRegion.doMiniBatchMutation(HRegion.java:3145)
  2143.         at org.apache.hadoop.hbase.regionserver.HRegion.batchMutate(HRegion.java:2793)
  2144.         at org.apache.hadoop.hbase.regionserver.HRegion.batchMutate(HRegion.java:2735)
  2145.         at org.apache.hadoop.hbase.regionserver.RSRpcServices.doBatchOp(RSRpcServices.java:692)
  2146.         at org.apache.hadoop.hbase.regionserver.RSRpcServices.doNonAtomicRegionMutation(RSRpcServices.java:654)
  2147.         at org.apache.hadoop.hbase.regionserver.RSRpcServices.multi(RSRpcServices.java:2029)
  2148.         at org.apache.hadoop.hbase.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:32213)
  2149.         at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:2112)
  2150.         at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:101)
  2151.         at org.apache.hadoop.hbase.ipc.RpcExecutor.consumerLoop(RpcExecutor.java:130)
  2152.         at org.apache.hadoop.hbase.ipc.RpcExecutor$1.run(RpcExecutor.java:107)
  2153.         at java.lang.Thread.run(Thread.java:745)
  2154.  
  2155.    Locked ownable synchronizers:
  2156.         - None
  2157.  
  2158. "B.defaultRpcServer.handler=1,queue=1,port=9104" #44 daemon prio=5 os_prio=0 tid=0x00007fbf715fa000 nid=0x5b in Object.wait() [0x00007fbf3fdf6000]
  2159.    java.lang.Thread.State: TIMED_WAITING (on object monitor)
  2160.         at java.lang.Object.wait(Native Method)
  2161.         at org.apache.hadoop.hbase.regionserver.wal.SyncFuture.get(SyncFuture.java:167)
  2162.         - locked <0x0000000548c571e0> (a org.apache.hadoop.hbase.regionserver.wal.SyncFuture)
  2163.         at org.apache.hadoop.hbase.regionserver.wal.FSHLog.blockOnSync(FSHLog.java:1472)
  2164.         at org.apache.hadoop.hbase.regionserver.wal.FSHLog.publishSyncThenBlockOnCompletion(FSHLog.java:1466)
  2165.         at org.apache.hadoop.hbase.regionserver.wal.FSHLog.sync(FSHLog.java:1600)
  2166.         at org.apache.hadoop.hbase.regionserver.HRegion.syncOrDefer(HRegion.java:7727)
  2167.         at org.apache.hadoop.hbase.regionserver.HRegion.doMiniBatchMutation(HRegion.java:3145)
  2168.         at org.apache.hadoop.hbase.regionserver.HRegion.batchMutate(HRegion.java:2793)
  2169.         at org.apache.hadoop.hbase.regionserver.HRegion.batchMutate(HRegion.java:2735)
  2170.         at org.apache.hadoop.hbase.regionserver.RSRpcServices.doBatchOp(RSRpcServices.java:692)
  2171.         at org.apache.hadoop.hbase.regionserver.RSRpcServices.doNonAtomicRegionMutation(RSRpcServices.java:654)
  2172.         at org.apache.hadoop.hbase.regionserver.RSRpcServices.multi(RSRpcServices.java:2029)
  2173.         at org.apache.hadoop.hbase.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:32213)
  2174.         at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:2112)
  2175.         at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:101)
  2176.         at org.apache.hadoop.hbase.ipc.RpcExecutor.consumerLoop(RpcExecutor.java:130)
  2177.         at org.apache.hadoop.hbase.ipc.RpcExecutor$1.run(RpcExecutor.java:107)
  2178.         at java.lang.Thread.run(Thread.java:745)
  2179.  
  2180.    Locked ownable synchronizers:
  2181.         - None
  2182.  
  2183. "B.defaultRpcServer.handler=0,queue=0,port=9104" #43 daemon prio=5 os_prio=0 tid=0x00007fbf715f8800 nid=0x5a in Object.wait() [0x00007fbf3fef7000]
  2184.    java.lang.Thread.State: TIMED_WAITING (on object monitor)
  2185.         at java.lang.Object.wait(Native Method)
  2186.         at org.apache.hadoop.hbase.regionserver.wal.SyncFuture.get(SyncFuture.java:167)
  2187.         - locked <0x0000000548c57208> (a org.apache.hadoop.hbase.regionserver.wal.SyncFuture)
  2188.         at org.apache.hadoop.hbase.regionserver.wal.FSHLog.blockOnSync(FSHLog.java:1472)
  2189.         at org.apache.hadoop.hbase.regionserver.wal.FSHLog.publishSyncThenBlockOnCompletion(FSHLog.java:1466)
  2190.         at org.apache.hadoop.hbase.regionserver.wal.FSHLog.sync(FSHLog.java:1600)
  2191.         at org.apache.hadoop.hbase.regionserver.HRegion.syncOrDefer(HRegion.java:7727)
  2192.         at org.apache.hadoop.hbase.regionserver.HRegion.doMiniBatchMutation(HRegion.java:3145)
  2193.         at org.apache.hadoop.hbase.regionserver.HRegion.batchMutate(HRegion.java:2793)
  2194.         at org.apache.hadoop.hbase.regionserver.HRegion.batchMutate(HRegion.java:2735)
  2195.         at org.apache.hadoop.hbase.regionserver.RSRpcServices.doBatchOp(RSRpcServices.java:692)
  2196.         at org.apache.hadoop.hbase.regionserver.RSRpcServices.doNonAtomicRegionMutation(RSRpcServices.java:654)
  2197.         at org.apache.hadoop.hbase.regionserver.RSRpcServices.multi(RSRpcServices.java:2029)
  2198.         at org.apache.hadoop.hbase.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:32213)
  2199.         at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:2112)
  2200.         at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:101)
  2201.         at org.apache.hadoop.hbase.ipc.RpcExecutor.consumerLoop(RpcExecutor.java:130)
  2202.         at org.apache.hadoop.hbase.ipc.RpcExecutor$1.run(RpcExecutor.java:107)
  2203.         at java.lang.Thread.run(Thread.java:745)
  2204.  
  2205.    Locked ownable synchronizers:
  2206.         - None
  2207.  
  2208. "RpcServer.listener,port=9104" #26 daemon prio=5 os_prio=0 tid=0x00007fbf71586000 nid=0x59 runnable [0x00007fbf3fff8000]
  2209.    java.lang.Thread.State: RUNNABLE
  2210.         at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method)
  2211.         at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269)
  2212.         at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:79)
  2213.         at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86)
  2214.         - locked <0x00000005480323e0> (a sun.nio.ch.Util$2)
  2215.         - locked <0x00000005480323f8> (a java.util.Collections$UnmodifiableSet)
  2216.         - locked <0x000000054802f978> (a sun.nio.ch.EPollSelectorImpl)
  2217.         at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97)
  2218.         at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:101)
  2219.         at org.apache.hadoop.hbase.ipc.RpcServer$Listener.run(RpcServer.java:730)
  2220.  
  2221.    Locked ownable synchronizers:
  2222.         - None
  2223.  
  2224. "RpcServer.responder" #38 daemon prio=5 os_prio=0 tid=0x00007fbf71585000 nid=0x58 runnable [0x00007fbf400f9000]
  2225.    java.lang.Thread.State: RUNNABLE
  2226.         at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method)
  2227.         at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269)
  2228.         at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:79)
  2229.         at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86)
  2230.         - locked <0x00000005485ef788> (a sun.nio.ch.Util$2)
  2231.         - locked <0x00000005485ef7a0> (a java.util.Collections$UnmodifiableSet)
  2232.         - locked <0x00000005485cf820> (a sun.nio.ch.EPollSelectorImpl)
  2233.         at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97)
  2234.         at org.apache.hadoop.hbase.ipc.RpcServer$Responder.doRunLoop(RpcServer.java:969)
  2235.         at org.apache.hadoop.hbase.ipc.RpcServer$Responder.run(RpcServer.java:916)
  2236.  
  2237.    Locked ownable synchronizers:
  2238.         - None
  2239.  
  2240. "main-EventThread" #42 daemon prio=5 os_prio=0 tid=0x00007fbf71598000 nid=0x57 waiting on condition [0x00007fbf401fa000]
  2241.    java.lang.Thread.State: WAITING (parking)
  2242.         at sun.misc.Unsafe.park(Native Method)
  2243.         - parking to wait for  <0x0000000548592e48> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject)
  2244.         at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)
  2245.         at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039)
  2246.         at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442)
  2247.         at org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:494)
  2248.  
  2249.    Locked ownable synchronizers:
  2250.         - None
  2251.  
  2252. "main-SendThread(r12s5.sjc.aristanetworks.com:2181)" #41 daemon prio=5 os_prio=0 tid=0x00007fbf715e9000 nid=0x56 runnable [0x00007fbf402fb000]
  2253.    java.lang.Thread.State: RUNNABLE
  2254.         at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method)
  2255.         at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269)
  2256.         at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:79)
  2257.         at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86)
  2258.         - locked <0x0000000548676dd8> (a sun.nio.ch.Util$2)
  2259.         - locked <0x0000000548676df0> (a java.util.Collections$UnmodifiableSet)
  2260.         - locked <0x00000005485bd610> (a sun.nio.ch.EPollSelectorImpl)
  2261.         at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97)
  2262.         at org.apache.zookeeper.ClientCnxnSocketNIO.doTransport(ClientCnxnSocketNIO.java:349)
  2263.         at org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1081)
  2264.  
  2265.    Locked ownable synchronizers:
  2266.         - None
  2267.  
  2268. "Thread-5" #39 daemon prio=5 os_prio=0 tid=0x00007fbf714ea800 nid=0x55 runnable [0x00007fbf403fc000]
  2269.    java.lang.Thread.State: RUNNABLE
  2270.         at org.apache.hadoop.net.unix.DomainSocketWatcher.doPoll0(Native Method)
  2271.         at org.apache.hadoop.net.unix.DomainSocketWatcher.access$800(DomainSocketWatcher.java:52)
  2272.         at org.apache.hadoop.net.unix.DomainSocketWatcher$1.run(DomainSocketWatcher.java:457)
  2273.         at java.lang.Thread.run(Thread.java:745)
  2274.  
  2275.    Locked ownable synchronizers:
  2276.         - None
  2277.  
  2278. "Timer for 'HBase' metrics system" #37 daemon prio=5 os_prio=0 tid=0x00007fbf7135f000 nid=0x54 in Object.wait() [0x00007fbf40afd000]
  2279.    java.lang.Thread.State: TIMED_WAITING (on object monitor)
  2280.         at java.lang.Object.wait(Native Method)
  2281.         at java.util.TimerThread.mainLoop(Timer.java:552)
  2282.         - locked <0x0000000548592dd0> (a java.util.TaskQueue)
  2283.         at java.util.TimerThread.run(Timer.java:505)
  2284.  
  2285.    Locked ownable synchronizers:
  2286.         - None
  2287.  
  2288. "RpcServer.reader=9,bindAddress=r12s16.sjc.aristanetworks.com,port=9104" #36 daemon prio=5 os_prio=0 tid=0x00007fbf7123f800 nid=0x53 runnable [0x00007fbf40bfe000]
  2289.    java.lang.Thread.State: RUNNABLE
  2290.         at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method)
  2291.         at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269)
  2292.         at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:79)
  2293.         at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86)
  2294.         - locked <0x0000000548036b90> (a sun.nio.ch.Util$2)
  2295.         - locked <0x0000000548036ba8> (a java.util.Collections$UnmodifiableSet)
  2296.         - locked <0x0000000548035cd8> (a sun.nio.ch.EPollSelectorImpl)
  2297.         at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97)
  2298.         at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:101)
  2299.         at org.apache.hadoop.hbase.ipc.RpcServer$Listener$Reader.doRunLoop(RpcServer.java:627)
  2300.         - locked <0x0000000548036bc0> (a org.apache.hadoop.hbase.ipc.RpcServer$Listener$Reader)
  2301.         at org.apache.hadoop.hbase.ipc.RpcServer$Listener$Reader.run(RpcServer.java:614)
  2302.         at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142)
  2303.         at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
  2304.         at java.lang.Thread.run(Thread.java:745)
  2305.  
  2306.    Locked ownable synchronizers:
  2307.         - <0x000000054804bea0> (a java.util.concurrent.ThreadPoolExecutor$Worker)
  2308.  
  2309. "RpcServer.reader=8,bindAddress=r12s16.sjc.aristanetworks.com,port=9104" #35 daemon prio=5 os_prio=0 tid=0x00007fbf7123e000 nid=0x52 runnable [0x00007fbf40cff000]
  2310.    java.lang.Thread.State: RUNNABLE
  2311.         at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method)
  2312.         at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269)
  2313.         at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:79)
  2314.         at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86)
  2315.         - locked <0x0000000548048c70> (a sun.nio.ch.Util$2)
  2316.         - locked <0x0000000548048c88> (a java.util.Collections$UnmodifiableSet)
  2317.         - locked <0x0000000548036758> (a sun.nio.ch.EPollSelectorImpl)
  2318.         at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97)
  2319.         at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:101)
  2320.         at org.apache.hadoop.hbase.ipc.RpcServer$Listener$Reader.doRunLoop(RpcServer.java:627)
  2321.         - locked <0x0000000548048ca0> (a org.apache.hadoop.hbase.ipc.RpcServer$Listener$Reader)
  2322.         at org.apache.hadoop.hbase.ipc.RpcServer$Listener$Reader.run(RpcServer.java:614)
  2323.         at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142)
  2324.         at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
  2325.         at java.lang.Thread.run(Thread.java:745)
  2326.  
  2327.    Locked ownable synchronizers:
  2328.         - <0x0000000548049870> (a java.util.concurrent.ThreadPoolExecutor$Worker)
  2329.  
  2330. "RpcServer.reader=7,bindAddress=r12s16.sjc.aristanetworks.com,port=9104" #34 daemon prio=5 os_prio=0 tid=0x00007fbf7123c000 nid=0x51 runnable [0x00007fbf40e00000]
  2331.    java.lang.Thread.State: RUNNABLE
  2332.         at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method)
  2333.         at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269)
  2334.         at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:79)
  2335.         at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86)
  2336.         - locked <0x0000000548028098> (a sun.nio.ch.Util$2)
  2337.         - locked <0x00000005480280b0> (a java.util.Collections$UnmodifiableSet)
  2338.         - locked <0x0000000548036158> (a sun.nio.ch.EPollSelectorImpl)
  2339.         at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97)
  2340.         at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:101)
  2341.         at org.apache.hadoop.hbase.ipc.RpcServer$Listener$Reader.doRunLoop(RpcServer.java:627)
  2342.         - locked <0x00000005480280c8> (a org.apache.hadoop.hbase.ipc.RpcServer$Listener$Reader)
  2343.         at org.apache.hadoop.hbase.ipc.RpcServer$Listener$Reader.run(RpcServer.java:614)
  2344.         at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142)
  2345.         at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
  2346.         at java.lang.Thread.run(Thread.java:745)
  2347.  
  2348.    Locked ownable synchronizers:
  2349.         - <0x0000000548036e90> (a java.util.concurrent.ThreadPoolExecutor$Worker)
  2350.  
  2351. "RpcServer.reader=6,bindAddress=r12s16.sjc.aristanetworks.com,port=9104" #33 daemon prio=5 os_prio=0 tid=0x00007fbf7123b000 nid=0x50 runnable [0x00007fbf40f01000]
  2352.    java.lang.Thread.State: RUNNABLE
  2353.         at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method)
  2354.         at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269)
  2355.         at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:79)
  2356.         at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86)
  2357.         - locked <0x00000005480280e0> (a sun.nio.ch.Util$2)
  2358.         - locked <0x00000005480280f8> (a java.util.Collections$UnmodifiableSet)
  2359.         - locked <0x0000000548036230> (a sun.nio.ch.EPollSelectorImpl)
  2360.         at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97)
  2361.         at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:101)
  2362.         at org.apache.hadoop.hbase.ipc.RpcServer$Listener$Reader.doRunLoop(RpcServer.java:627)
  2363.         - locked <0x0000000548028110> (a org.apache.hadoop.hbase.ipc.RpcServer$Listener$Reader)
  2364.         at org.apache.hadoop.hbase.ipc.RpcServer$Listener$Reader.run(RpcServer.java:614)
  2365.         at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142)
  2366.         at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
  2367.         at java.lang.Thread.run(Thread.java:745)
  2368.  
  2369.    Locked ownable synchronizers:
  2370.         - <0x0000000548036ec0> (a java.util.concurrent.ThreadPoolExecutor$Worker)
  2371.  
  2372. "RpcServer.reader=5,bindAddress=r12s16.sjc.aristanetworks.com,port=9104" #32 daemon prio=5 os_prio=0 tid=0x00007fbf711ab800 nid=0x4f runnable [0x00007fbf41002000]
  2373.    java.lang.Thread.State: RUNNABLE
  2374.         at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method)
  2375.         at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269)
  2376.         at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:79)
  2377.         at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86)
  2378.         - locked <0x0000000548028128> (a sun.nio.ch.Util$2)
  2379.         - locked <0x0000000548028140> (a java.util.Collections$UnmodifiableSet)
  2380.         - locked <0x00000005480362c0> (a sun.nio.ch.EPollSelectorImpl)
  2381.         at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97)
  2382.         at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:101)
  2383.         at org.apache.hadoop.hbase.ipc.RpcServer$Listener$Reader.doRunLoop(RpcServer.java:627)
  2384.         - locked <0x0000000548028158> (a org.apache.hadoop.hbase.ipc.RpcServer$Listener$Reader)
  2385.         at org.apache.hadoop.hbase.ipc.RpcServer$Listener$Reader.run(RpcServer.java:614)
  2386.         at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142)
  2387.         at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
  2388.         at java.lang.Thread.run(Thread.java:745)
  2389.  
  2390.    Locked ownable synchronizers:
  2391.         - <0x0000000548036ef0> (a java.util.concurrent.ThreadPoolExecutor$Worker)
  2392.  
  2393. "RpcServer.reader=4,bindAddress=r12s16.sjc.aristanetworks.com,port=9104" #31 daemon prio=5 os_prio=0 tid=0x00007fbf711aa000 nid=0x4e runnable [0x00007fbf41103000]
  2394.    java.lang.Thread.State: RUNNABLE
  2395.         at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method)
  2396.         at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269)
  2397.         at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:79)
  2398.         at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86)
  2399.         - locked <0x000000054803c290> (a sun.nio.ch.Util$2)
  2400.         - locked <0x000000054803c2a8> (a java.util.Collections$UnmodifiableSet)
  2401.         - locked <0x000000054803b158> (a sun.nio.ch.EPollSelectorImpl)
  2402.         at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97)
  2403.         at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:101)
  2404.         at org.apache.hadoop.hbase.ipc.RpcServer$Listener$Reader.doRunLoop(RpcServer.java:627)
  2405.         - locked <0x000000054803c2c0> (a org.apache.hadoop.hbase.ipc.RpcServer$Listener$Reader)
  2406.         at org.apache.hadoop.hbase.ipc.RpcServer$Listener$Reader.run(RpcServer.java:614)
  2407.         at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142)
  2408.         at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
  2409.         at java.lang.Thread.run(Thread.java:745)
  2410.  
  2411.    Locked ownable synchronizers:
  2412.         - <0x000000054803c410> (a java.util.concurrent.ThreadPoolExecutor$Worker)
  2413.  
  2414. "RpcServer.reader=3,bindAddress=r12s16.sjc.aristanetworks.com,port=9104" #30 daemon prio=5 os_prio=0 tid=0x00007fbf711a8000 nid=0x4d runnable [0x00007fbf41204000]
  2415.    java.lang.Thread.State: RUNNABLE
  2416.         at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method)
  2417.         at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269)
  2418.         at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:79)
  2419.         at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86)
  2420.         - locked <0x00000005480226c0> (a sun.nio.ch.Util$2)
  2421.         - locked <0x00000005480226d8> (a java.util.Collections$UnmodifiableSet)
  2422.         - locked <0x000000054803b758> (a sun.nio.ch.EPollSelectorImpl)
  2423.         at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97)
  2424.         at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:101)
  2425.         at org.apache.hadoop.hbase.ipc.RpcServer$Listener$Reader.doRunLoop(RpcServer.java:627)
  2426.         - locked <0x00000005480226f0> (a org.apache.hadoop.hbase.ipc.RpcServer$Listener$Reader)
  2427.         at org.apache.hadoop.hbase.ipc.RpcServer$Listener$Reader.run(RpcServer.java:614)
  2428.         at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142)
  2429.         at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
  2430.         at java.lang.Thread.run(Thread.java:745)
  2431.  
  2432.    Locked ownable synchronizers:
  2433.         - <0x000000054803bf90> (a java.util.concurrent.ThreadPoolExecutor$Worker)
  2434.  
  2435. "RpcServer.reader=2,bindAddress=r12s16.sjc.aristanetworks.com,port=9104" #29 daemon prio=5 os_prio=0 tid=0x00007fbf711a6800 nid=0x4c runnable [0x00007fbf41305000]
  2436.    java.lang.Thread.State: RUNNABLE
  2437.         at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method)
  2438.         at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269)
  2439.         at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:79)
  2440.         at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86)
  2441.         - locked <0x0000000548028170> (a sun.nio.ch.Util$2)
  2442.         - locked <0x0000000548028188> (a java.util.Collections$UnmodifiableSet)
  2443.         - locked <0x0000000548036350> (a sun.nio.ch.EPollSelectorImpl)
  2444.         at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97)
  2445.         at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:101)
  2446.         at org.apache.hadoop.hbase.ipc.RpcServer$Listener$Reader.doRunLoop(RpcServer.java:627)
  2447.         - locked <0x00000005480281a0> (a org.apache.hadoop.hbase.ipc.RpcServer$Listener$Reader)
  2448.         at org.apache.hadoop.hbase.ipc.RpcServer$Listener$Reader.run(RpcServer.java:614)
  2449.         at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142)
  2450.         at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
  2451.         at java.lang.Thread.run(Thread.java:745)
  2452.  
  2453.    Locked ownable synchronizers:
  2454.         - <0x0000000548036f20> (a java.util.concurrent.ThreadPoolExecutor$Worker)
  2455.  
  2456. "RpcServer.reader=1,bindAddress=r12s16.sjc.aristanetworks.com,port=9104" #28 daemon prio=5 os_prio=0 tid=0x00007fbf711bf800 nid=0x4b runnable [0x00007fbf41406000]
  2457.    java.lang.Thread.State: RUNNABLE
  2458.         at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method)
  2459.         at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269)
  2460.         at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:79)
  2461.         at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86)
  2462.         - locked <0x00000005480281b8> (a sun.nio.ch.Util$2)
  2463.         - locked <0x000000054803be10> (a java.util.Collections$UnmodifiableSet)
  2464.         - locked <0x00000005480363e0> (a sun.nio.ch.EPollSelectorImpl)
  2465.         at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97)
  2466.         at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:101)
  2467.         at org.apache.hadoop.hbase.ipc.RpcServer$Listener$Reader.doRunLoop(RpcServer.java:627)
  2468.         - locked <0x000000054803be28> (a org.apache.hadoop.hbase.ipc.RpcServer$Listener$Reader)
  2469.         at org.apache.hadoop.hbase.ipc.RpcServer$Listener$Reader.run(RpcServer.java:614)
  2470.         at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142)
  2471.         at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
  2472.         at java.lang.Thread.run(Thread.java:745)
  2473.  
  2474.    Locked ownable synchronizers:
  2475.         - <0x0000000548036f50> (a java.util.concurrent.ThreadPoolExecutor$Worker)
  2476.  
  2477. "RpcServer.reader=0,bindAddress=r12s16.sjc.aristanetworks.com,port=9104" #27 daemon prio=5 os_prio=0 tid=0x00007fbf711be800 nid=0x4a runnable [0x00007fbf41d2e000]
  2478.    java.lang.Thread.State: RUNNABLE
  2479.         at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method)
  2480.         at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269)
  2481.         at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:79)
  2482.         at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86)
  2483.         - locked <0x000000054803be40> (a sun.nio.ch.Util$2)
  2484.         - locked <0x000000054803be58> (a java.util.Collections$UnmodifiableSet)
  2485.         - locked <0x0000000548036470> (a sun.nio.ch.EPollSelectorImpl)
  2486.         at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97)
  2487.         at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:101)
  2488.         at org.apache.hadoop.hbase.ipc.RpcServer$Listener$Reader.doRunLoop(RpcServer.java:627)
  2489.         - locked <0x000000054803be70> (a org.apache.hadoop.hbase.ipc.RpcServer$Listener$Reader)
  2490.         at org.apache.hadoop.hbase.ipc.RpcServer$Listener$Reader.run(RpcServer.java:614)
  2491.         at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142)
  2492.         at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
  2493.         at java.lang.Thread.run(Thread.java:745)
  2494.  
  2495.    Locked ownable synchronizers:
  2496.         - <0x0000000548036f80> (a java.util.concurrent.ThreadPoolExecutor$Worker)
  2497.  
  2498. "RMI TCP Accept-0" #22 daemon prio=5 os_prio=0 tid=0x00007fbf70988000 nid=0x45 runnable [0x00007fbf42369000]
  2499.    java.lang.Thread.State: RUNNABLE
  2500.         at java.net.PlainSocketImpl.socketAccept(Native Method)
  2501.         at java.net.AbstractPlainSocketImpl.accept(AbstractPlainSocketImpl.java:404)
  2502.         at java.net.ServerSocket.implAccept(ServerSocket.java:545)
  2503.         at java.net.ServerSocket.accept(ServerSocket.java:513)
  2504.         at sun.management.jmxremote.LocalRMIServerSocketFactory$1.accept(LocalRMIServerSocketFactory.java:52)
  2505.         at sun.rmi.transport.tcp.TCPTransport$AcceptLoop.executeAcceptLoop(TCPTransport.java:400)
  2506.         at sun.rmi.transport.tcp.TCPTransport$AcceptLoop.run(TCPTransport.java:372)
  2507.         at java.lang.Thread.run(Thread.java:745)
  2508.  
  2509.    Locked ownable synchronizers:
  2510.         - None
  2511.  
  2512. "RMI TCP Accept-11102" #21 daemon prio=5 os_prio=0 tid=0x00007fbf7097f800 nid=0x44 runnable [0x00007fbf4246a000]
  2513.    java.lang.Thread.State: RUNNABLE
  2514.         at java.net.PlainSocketImpl.socketAccept(Native Method)
  2515.         at java.net.AbstractPlainSocketImpl.accept(AbstractPlainSocketImpl.java:404)
  2516.         at java.net.ServerSocket.implAccept(ServerSocket.java:545)
  2517.         at java.net.ServerSocket.accept(ServerSocket.java:513)
  2518.         at sun.rmi.transport.tcp.TCPTransport$AcceptLoop.executeAcceptLoop(TCPTransport.java:400)
  2519.         at sun.rmi.transport.tcp.TCPTransport$AcceptLoop.run(TCPTransport.java:372)
  2520.         at java.lang.Thread.run(Thread.java:745)
  2521.  
  2522.    Locked ownable synchronizers:
  2523.         - None
  2524.  
  2525. "RMI TCP Accept-0" #20 daemon prio=5 os_prio=0 tid=0x00007fbf70967800 nid=0x43 runnable [0x00007fbf4256b000]
  2526.    java.lang.Thread.State: RUNNABLE
  2527.         at java.net.PlainSocketImpl.socketAccept(Native Method)
  2528.         at java.net.AbstractPlainSocketImpl.accept(AbstractPlainSocketImpl.java:404)
  2529.         at java.net.ServerSocket.implAccept(ServerSocket.java:545)
  2530.         at java.net.ServerSocket.accept(ServerSocket.java:513)
  2531.         at sun.rmi.transport.tcp.TCPTransport$AcceptLoop.executeAcceptLoop(TCPTransport.java:400)
  2532.         at sun.rmi.transport.tcp.TCPTransport$AcceptLoop.run(TCPTransport.java:372)
  2533.         at java.lang.Thread.run(Thread.java:745)
  2534.  
  2535.    Locked ownable synchronizers:
  2536.         - None
  2537.  
  2538. "Service Thread" #18 daemon prio=9 os_prio=0 tid=0x00007fbf702f0000 nid=0x42 runnable [0x0000000000000000]
  2539.    java.lang.Thread.State: RUNNABLE
  2540.  
  2541.    Locked ownable synchronizers:
  2542.         - None
  2543.  
  2544. "C1 CompilerThread11" #17 daemon prio=9 os_prio=0 tid=0x00007fbf702e4000 nid=0x41 waiting on condition [0x0000000000000000]
  2545.    java.lang.Thread.State: RUNNABLE
  2546.  
  2547.    Locked ownable synchronizers:
  2548.         - None
  2549.  
  2550. "C1 CompilerThread10" #16 daemon prio=9 os_prio=0 tid=0x00007fbf702da000 nid=0x40 waiting on condition [0x0000000000000000]
  2551.    java.lang.Thread.State: RUNNABLE
  2552.  
  2553.    Locked ownable synchronizers:
  2554.         - None
  2555.  
  2556. "C1 CompilerThread9" #15 daemon prio=9 os_prio=0 tid=0x00007fbf702d8000 nid=0x3f waiting on condition [0x0000000000000000]
  2557.    java.lang.Thread.State: RUNNABLE
  2558.  
  2559.    Locked ownable synchronizers:
  2560.         - None
  2561.  
  2562. "C1 CompilerThread8" #14 daemon prio=9 os_prio=0 tid=0x00007fbf702d5000 nid=0x3e waiting on condition [0x0000000000000000]
  2563.    java.lang.Thread.State: RUNNABLE
  2564.  
  2565.    Locked ownable synchronizers:
  2566.         - None
  2567.  
  2568. "C2 CompilerThread7" #13 daemon prio=9 os_prio=0 tid=0x00007fbf702d3000 nid=0x3d waiting on condition [0x0000000000000000]
  2569.    java.lang.Thread.State: RUNNABLE
  2570.  
  2571.    Locked ownable synchronizers:
  2572.         - None
  2573.  
  2574. "C2 CompilerThread6" #12 daemon prio=9 os_prio=0 tid=0x00007fbf702d0800 nid=0x3c waiting on condition [0x0000000000000000]
  2575.    java.lang.Thread.State: RUNNABLE
  2576.  
  2577.    Locked ownable synchronizers:
  2578.         - None
  2579.  
  2580. "C2 CompilerThread5" #11 daemon prio=9 os_prio=0 tid=0x00007fbf702ce800 nid=0x3b waiting on condition [0x0000000000000000]
  2581.    java.lang.Thread.State: RUNNABLE
  2582.  
  2583.    Locked ownable synchronizers:
  2584.         - None
  2585.  
  2586. "C2 CompilerThread4" #10 daemon prio=9 os_prio=0 tid=0x00007fbf702cc800 nid=0x3a waiting on condition [0x0000000000000000]
  2587.    java.lang.Thread.State: RUNNABLE
  2588.  
  2589.    Locked ownable synchronizers:
  2590.         - None
  2591.  
  2592. "C2 CompilerThread3" #9 daemon prio=9 os_prio=0 tid=0x00007fbf702c2800 nid=0x39 waiting on condition [0x0000000000000000]
  2593.    java.lang.Thread.State: RUNNABLE
  2594.  
  2595.    Locked ownable synchronizers:
  2596.         - None
  2597.  
  2598. "C2 CompilerThread2" #8 daemon prio=9 os_prio=0 tid=0x00007fbf702b8000 nid=0x38 waiting on condition [0x0000000000000000]
  2599.    java.lang.Thread.State: RUNNABLE
  2600.  
  2601.    Locked ownable synchronizers:
  2602.         - None
  2603.  
  2604. "C2 CompilerThread1" #7 daemon prio=9 os_prio=0 tid=0x00007fbf702b3800 nid=0x37 waiting on condition [0x0000000000000000]
  2605.    java.lang.Thread.State: RUNNABLE
  2606.  
  2607.    Locked ownable synchronizers:
  2608.         - None
  2609.  
  2610. "C2 CompilerThread0" #6 daemon prio=9 os_prio=0 tid=0x00007fbf702b1800 nid=0x36 waiting on condition [0x0000000000000000]
  2611.    java.lang.Thread.State: RUNNABLE
  2612.  
  2613.    Locked ownable synchronizers:
  2614.         - None
  2615.  
  2616. "Signal Dispatcher" #5 daemon prio=9 os_prio=0 tid=0x00007fbf702af000 nid=0x35 runnable [0x0000000000000000]
  2617.    java.lang.Thread.State: RUNNABLE
  2618.  
  2619.    Locked ownable synchronizers:
  2620.         - None
  2621.  
  2622. "Surrogate Locker Thread (Concurrent GC)" #4 daemon prio=9 os_prio=0 tid=0x00007fbf702ad000 nid=0x34 waiting on condition [0x0000000000000000]
  2623.    java.lang.Thread.State: RUNNABLE
  2624.  
  2625.    Locked ownable synchronizers:
  2626.         - None
  2627.  
  2628. "Finalizer" #3 daemon prio=8 os_prio=0 tid=0x00007fbf70283800 nid=0x33 in Object.wait() [0x00007fbf4732b000]
  2629.    java.lang.Thread.State: WAITING (on object monitor)
  2630.         at java.lang.Object.wait(Native Method)
  2631.         at java.lang.ref.ReferenceQueue.remove(ReferenceQueue.java:143)
  2632.         - locked <0x000000054803be88> (a java.lang.ref.ReferenceQueue$Lock)
  2633.         at java.lang.ref.ReferenceQueue.remove(ReferenceQueue.java:164)
  2634.         at java.lang.ref.Finalizer$FinalizerThread.run(Finalizer.java:209)
  2635.  
  2636.    Locked ownable synchronizers:
  2637.         - None
  2638.  
  2639. "Reference Handler" #2 daemon prio=10 os_prio=0 tid=0x00007fbf70281800 nid=0x32 in Object.wait() [0x00007fbf4742c000]
  2640.    java.lang.Thread.State: WAITING (on object monitor)
  2641.         at java.lang.Object.wait(Native Method)
  2642.         at java.lang.Object.wait(Object.java:502)
  2643.         at java.lang.ref.Reference$ReferenceHandler.run(Reference.java:157)
  2644.         - locked <0x0000000548032428> (a java.lang.ref.Reference$Lock)
  2645.  
  2646.    Locked ownable synchronizers:
  2647.         - None
  2648.  
  2649. "main" #1 prio=5 os_prio=0 tid=0x00007fbf7000e000 nid=0x18 in Object.wait() [0x00007fbf78f4d000]
  2650.    java.lang.Thread.State: WAITING (on object monitor)
  2651.         at java.lang.Object.wait(Native Method)
  2652.         at java.lang.Thread.join(Thread.java:1245)
  2653.         - locked <0x00000005483a67b0> (a java.lang.Thread)
  2654.         at java.lang.Thread.join(Thread.java:1319)
  2655.         at org.apache.hadoop.hbase.util.HasThread.join(HasThread.java:89)
  2656.         at org.apache.hadoop.hbase.regionserver.HRegionServerCommandLine.start(HRegionServerCommandLine.java:66)
  2657.         at org.apache.hadoop.hbase.regionserver.HRegionServerCommandLine.run(HRegionServerCommandLine.java:87)
  2658.         at org.apache.hadoop.util.ToolRunner.run(ToolRunner.java:70)
  2659.         at org.apache.hadoop.hbase.util.ServerCommandLine.doMain(ServerCommandLine.java:126)
  2660.         at org.apache.hadoop.hbase.regionserver.HRegionServer.main(HRegionServer.java:2641)
  2661.  
  2662.    Locked ownable synchronizers:
  2663.         - None
  2664.  
  2665. "VM Thread" os_prio=0 tid=0x00007fbf7027c800 nid=0x31 runnable
  2666.  
  2667. "Gang worker#0 (Parallel GC Threads)" os_prio=0 tid=0x00007fbf7001f000 nid=0x19 runnable
  2668.  
  2669. "Gang worker#1 (Parallel GC Threads)" os_prio=0 tid=0x00007fbf70020800 nid=0x1a runnable
  2670.  
  2671. "Gang worker#2 (Parallel GC Threads)" os_prio=0 tid=0x00007fbf70022800 nid=0x1b runnable
  2672.  
  2673. "Gang worker#3 (Parallel GC Threads)" os_prio=0 tid=0x00007fbf70024000 nid=0x1c runnable
  2674.  
  2675. "Gang worker#4 (Parallel GC Threads)" os_prio=0 tid=0x00007fbf70026000 nid=0x1d runnable
  2676.  
  2677. "Gang worker#5 (Parallel GC Threads)" os_prio=0 tid=0x00007fbf70027800 nid=0x1e runnable
  2678.  
  2679. "Gang worker#6 (Parallel GC Threads)" os_prio=0 tid=0x00007fbf70029800 nid=0x1f runnable
  2680.  
  2681. "Gang worker#7 (Parallel GC Threads)" os_prio=0 tid=0x00007fbf7002b000 nid=0x20 runnable
  2682.  
  2683. "Gang worker#8 (Parallel GC Threads)" os_prio=0 tid=0x00007fbf7002d000 nid=0x21 runnable
  2684.  
  2685. "Gang worker#9 (Parallel GC Threads)" os_prio=0 tid=0x00007fbf7002e800 nid=0x22 runnable
  2686.  
  2687. "Gang worker#10 (Parallel GC Threads)" os_prio=0 tid=0x00007fbf70030800 nid=0x23 runnable
  2688.  
  2689. "Gang worker#11 (Parallel GC Threads)" os_prio=0 tid=0x00007fbf70032000 nid=0x24 runnable
  2690.  
  2691. "Gang worker#12 (Parallel GC Threads)" os_prio=0 tid=0x00007fbf70034000 nid=0x25 runnable
  2692.  
  2693. "Gang worker#13 (Parallel GC Threads)" os_prio=0 tid=0x00007fbf70035800 nid=0x26 runnable
  2694.  
  2695. "Gang worker#14 (Parallel GC Threads)" os_prio=0 tid=0x00007fbf70037800 nid=0x27 runnable
  2696.  
  2697. "Gang worker#15 (Parallel GC Threads)" os_prio=0 tid=0x00007fbf70039000 nid=0x28 runnable
  2698.  
  2699. "Gang worker#16 (Parallel GC Threads)" os_prio=0 tid=0x00007fbf7003b000 nid=0x29 runnable
  2700.  
  2701. "Gang worker#17 (Parallel GC Threads)" os_prio=0 tid=0x00007fbf7003c800 nid=0x2a runnable
  2702.  
  2703. "Concurrent Mark-Sweep GC Thread" os_prio=0 tid=0x00007fbf7011d800 nid=0x30 runnable
  2704.  
  2705. "Gang worker#0 (Parallel CMS Threads)" os_prio=0 tid=0x00007fbf70114000 nid=0x2b runnable
  2706.  
  2707. "Gang worker#1 (Parallel CMS Threads)" os_prio=0 tid=0x00007fbf70116000 nid=0x2c runnable
  2708.  
  2709. "Gang worker#2 (Parallel CMS Threads)" os_prio=0 tid=0x00007fbf70117800 nid=0x2d runnable
  2710.  
  2711. "Gang worker#3 (Parallel CMS Threads)" os_prio=0 tid=0x00007fbf70119000 nid=0x2e runnable
  2712.  
  2713. "Gang worker#4 (Parallel CMS Threads)" os_prio=0 tid=0x00007fbf7011b000 nid=0x2f runnable
  2714.  
  2715. "VM Periodic Task Thread" os_prio=0 tid=0x00007fbf7098c800 nid=0x46 waiting on condition
  2716.  
  2717. JNI global references: 347
clone this paste RAW Paste Data