[22/Nov/2022 08:45:38 +0000] hbase-REGIONSERVER-c5c018d81a72ab9f0e37aac51c1a8f83-jstack-stacks-collector 7.6.2__COLLECTION_DURATION__ 465 __BEGIN_STACKS_SAMPLE__ 2022-11-22 08:45:37 Full thread dump OpenJDK 64-Bit Server VM (25.352-b08 mixed mode): "ForkJoinPool.commonPool-worker-4" #577921 daemon prio=5 os_prio=0 tid=0x00007fc00f0b1800 nid=0x2286 waiting on condition [0x00007fc0e0ed1000] java.lang.Thread.State: WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007fc1dc0900a8> (a java.util.concurrent.ForkJoinPool) at java.util.concurrent.ForkJoinPool.awaitWork(ForkJoinPool.java:1824) at java.util.concurrent.ForkJoinPool.runWorker(ForkJoinPool.java:1693) at java.util.concurrent.ForkJoinWorkerThread.run(ForkJoinWorkerThread.java:175) "ForkJoinPool.commonPool-worker-5" #577913 daemon prio=5 os_prio=0 tid=0x00007fc00edfb000 nid=0x2247 waiting on condition [0x00007fc05afee000] java.lang.Thread.State: WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007fc1dc0900a8> (a java.util.concurrent.ForkJoinPool) at java.util.concurrent.ForkJoinPool.awaitWork(ForkJoinPool.java:1824) at java.util.concurrent.ForkJoinPool.runWorker(ForkJoinPool.java:1693) at java.util.concurrent.ForkJoinWorkerThread.run(ForkJoinWorkerThread.java:175) "ForkJoinPool.commonPool-worker-2" #577905 daemon prio=5 os_prio=0 tid=0x00007fc01751e800 nid=0x2211 waiting on condition [0x00007fc0c1001000] java.lang.Thread.State: WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007fc1dc0900a8> (a java.util.concurrent.ForkJoinPool) at java.util.concurrent.ForkJoinPool.awaitWork(ForkJoinPool.java:1824) at java.util.concurrent.ForkJoinPool.runWorker(ForkJoinPool.java:1693) at java.util.concurrent.ForkJoinWorkerThread.run(ForkJoinWorkerThread.java:175) "ForkJoinPool.commonPool-worker-3" #577899 daemon prio=5 os_prio=0 tid=0x00007fc011834800 nid=0x21e8 waiting on condition [0x00007fc0501c0000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007fc1dc0900a8> (a java.util.concurrent.ForkJoinPool) at java.util.concurrent.ForkJoinPool.awaitWork(ForkJoinPool.java:1824) at java.util.concurrent.ForkJoinPool.runWorker(ForkJoinPool.java:1693) at java.util.concurrent.ForkJoinWorkerThread.run(ForkJoinWorkerThread.java:175) "ForkJoinPool.commonPool-worker-6" #577890 daemon prio=5 os_prio=0 tid=0x00007fc01752d000 nid=0x218b waiting on condition [0x00007fc0c61e0000] java.lang.Thread.State: WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007fc1dc0900a8> (a java.util.concurrent.ForkJoinPool) at java.util.concurrent.ForkJoinPool.awaitWork(ForkJoinPool.java:1824) at java.util.concurrent.ForkJoinPool.runWorker(ForkJoinPool.java:1693) at java.util.concurrent.ForkJoinWorkerThread.run(ForkJoinWorkerThread.java:175) "ForkJoinPool.commonPool-worker-0" #577880 daemon prio=5 os_prio=0 tid=0x00007fc011605000 nid=0x2134 waiting on condition [0x00007fc07b2f1000] java.lang.Thread.State: WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007fc1dc0900a8> (a java.util.concurrent.ForkJoinPool) at java.util.concurrent.ForkJoinPool.awaitWork(ForkJoinPool.java:1824) at java.util.concurrent.ForkJoinPool.runWorker(ForkJoinPool.java:1693) at java.util.concurrent.ForkJoinWorkerThread.run(ForkJoinWorkerThread.java:175) "s3a-transfer-my-bucket-dl-unbounded-pool2-t50167" #577868 daemon prio=5 os_prio=0 tid=0x00007fc011831000 nid=0x20c2 waiting on condition [0x00007fc0bb9f8000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007fc1c9fe5a20> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:750) "s3a-transfer-my-bucket-dl-bounded-pool1-t35445" #577867 daemon prio=5 os_prio=0 tid=0x0000558b3929a800 nid=0x20c1 waiting on condition [0x00007fc0c69e8000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007fc1dbeb34c8> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:750) "s3a-transfer-my-bucket-dl-unbounded-pool2-t50166" #577866 daemon prio=5 os_prio=0 tid=0x0000558b3929a000 nid=0x20be waiting on condition [0x00007fc0c66e5000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007fc1c9fe5a20> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:750) "s3a-transfer-my-bucket-dl-bounded-pool1-t35444" #577865 daemon prio=5 os_prio=0 tid=0x0000558b39299000 nid=0x20bd waiting on condition [0x00007fc0c2f20000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007fc1dbeb34c8> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:750) "process reaper" #577863 daemon prio=10 os_prio=0 tid=0x0000558b39297800 nid=0x20b9 waiting on condition [0x00007fc10b139000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007fc1bc11a708> (a java.util.concurrent.SynchronousQueue$TransferStack) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460) at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362) at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:750) "ForkJoinPool.commonPool-worker-7" #577853 daemon prio=5 os_prio=0 tid=0x00007fc03cb17800 nid=0x204a waiting on condition [0x00007fc0c0dff000] java.lang.Thread.State: WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007fc1dc0900a8> (a java.util.concurrent.ForkJoinPool) at java.util.concurrent.ForkJoinPool.awaitWork(ForkJoinPool.java:1824) at java.util.concurrent.ForkJoinPool.runWorker(ForkJoinPool.java:1693) at java.util.concurrent.ForkJoinWorkerThread.run(ForkJoinWorkerThread.java:175) "Timer for 'HBase' metrics system" #577711 daemon prio=5 os_prio=0 tid=0x00007fc0182c2800 nid=0x1b9e in Object.wait() [0x00007fc0c3829000] java.lang.Thread.State: TIMED_WAITING (on object monitor) at java.lang.Object.wait(Native Method) at java.util.TimerThread.mainLoop(Timer.java:552) - locked <0x00007fc6b1395380> (a java.util.TaskQueue) at java.util.TimerThread.run(Timer.java:505) "ResponseProcessor for block BP-697380007-10.154.203.225-1667554502541:blk_1074888498_1147827" #577596 daemon prio=5 os_prio=0 tid=0x00007fc011850800 nid=0x1861 runnable [0x00007fc0513d2000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007fc6ace278a0> (a sun.nio.ch.Util$3) - locked <0x00007fc6ace27890> (a java.util.Collections$UnmodifiableSet) - locked <0x00007fc6ace27668> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:336) at org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:157) at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:118) at java.io.FilterInputStream.read(FilterInputStream.java:83) at java.io.FilterInputStream.read(FilterInputStream.java:83) at org.apache.hadoop.hdfs.protocolPB.PBHelperClient.vintPrefixed(PBHelperClient.java:550) at org.apache.hadoop.hdfs.protocol.datatransfer.PipelineAck.readFields(PipelineAck.java:213) at org.apache.hadoop.hdfs.DataStreamer$ResponseProcessor.run(DataStreamer.java:1101) "DataStreamer for file /hbase-wals/WALs/cod-1e3xsc06g8mjm-worker124.exait3p.z1did1.b0.cloudera.site,16020,1668705752838/cod-1e3xsc06g8mjm-worker124.exait3p.z1did1.b0.cloudera.site%2C16020%2C1668705752838.cod-1e3xsc06g8mjm-worker124.exait3p.z1did1.b0.cloudera.site%2C16020%2C1668705752838.regiongroup-0.1669106503759 block BP-697380007-10.154.203.225-1667554502541:blk_1074888498_1147827" #577595 daemon prio=5 os_prio=0 tid=0x00007fc03cb1a800 nid=0x185e in Object.wait() [0x00007fc0512d1000] java.lang.Thread.State: TIMED_WAITING (on object monitor) at java.lang.Object.wait(Native Method) at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:681) - locked <0x00007fc6a2b0ec38> (a java.util.LinkedList) "SinkThread-7427" #576980 daemon prio=5 os_prio=0 tid=0x00007fc00d1c4000 nid=0x444 waiting on condition [0x00007fc0c3627000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007fc2b7f24050> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:750) "SinkThread-7426" #576978 daemon prio=5 os_prio=0 tid=0x00007fc00d1c3800 nid=0x43c in Object.wait() [0x00007fc0e9e60000] java.lang.Thread.State: WAITING (on object monitor) at java.lang.Object.wait(Native Method) at java.lang.Object.wait(Object.java:502) at org.apache.hadoop.hbase.ipc.BlockingRpcCallback.get(BlockingRpcCallback.java:62) - locked <0x00007fc912e8e3f0> (a org.apache.hadoop.hbase.ipc.BlockingRpcCallback) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callBlockingMethod(AbstractRpcClient.java:328) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$200(AbstractRpcClient.java:89) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$BlockingRpcChannelImplementation.callBlockingMethod(AbstractRpcClient.java:576) at org.apache.hadoop.hbase.shaded.protobuf.generated.AdminProtos$AdminService$BlockingStub.replicateWALEntry(AdminProtos.java:36789) at org.apache.hadoop.hbase.protobuf.ReplicationProtbufUtil.replicateWALEntry(ReplicationProtbufUtil.java:67) at org.apache.hadoop.hbase.replication.regionserver.HBaseInterClusterReplicationEndpoint.replicateEntries(HBaseInterClusterReplicationEndpoint.java:634) at org.apache.hadoop.hbase.replication.regionserver.HBaseInterClusterReplicationEndpoint.lambda$createReplicator$3(HBaseInterClusterReplicationEndpoint.java:677) at org.apache.hadoop.hbase.replication.regionserver.HBaseInterClusterReplicationEndpoint$$Lambda$714/736574045.call(Unknown Source) at java.util.concurrent.FutureTask.run(FutureTask.java:266) at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:511) at java.util.concurrent.FutureTask.run(FutureTask.java:266) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:750) "SinkThread-7425" #576976 daemon prio=5 os_prio=0 tid=0x00007fc00e92d000 nid=0x436 waiting on condition [0x00007fc0c1c0d000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007fc2b7f24050> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:750) "SinkThread-7424" #576973 daemon prio=5 os_prio=0 tid=0x00007fc00e92c000 nid=0x42a waiting on condition [0x00007fc0e9a5c000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007fc2b7f24050> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:750) "SinkThread-7423" #576971 daemon prio=5 os_prio=0 tid=0x00007fc00e92b800 nid=0x408 waiting on condition [0x00007fc0c0f00000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007fc2b7f24050> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:750) "SinkThread-7422" #576968 daemon prio=5 os_prio=0 tid=0x00007fc00e92a800 nid=0x3fd waiting on condition [0x00007fc0c291a000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007fc2b7f24050> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:750) "SinkThread-7421" #576962 daemon prio=5 os_prio=0 tid=0x00007fc00c5c7000 nid=0x3d2 waiting on condition [0x00007fc0cf8f9000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007fc2b7f24050> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:750) "SinkThread-7420" #576958 daemon prio=5 os_prio=0 tid=0x00007fc00c5c6000 nid=0x3ca waiting on condition [0x00007fc0787c6000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007fc2b7f24050> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:750) "SinkThread-7419" #576920 daemon prio=5 os_prio=0 tid=0x00007fc00e93d800 nid=0x294 waiting on condition [0x00007fc05a9e8000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007fc2b7f24050> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:750) "SinkThread-7418" #576913 daemon prio=5 os_prio=0 tid=0x00007fc00e933000 nid=0x245 waiting on condition [0x00007fc0b94d3000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007fc2b7f24050> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:750) "Thread-287977" #570389 daemon prio=5 os_prio=0 tid=0x00007fc03da0e800 nid=0x3bf8 waiting on condition [0x00007fc0c2213000] java.lang.Thread.State: TIMED_WAITING (sleeping) at java.lang.Thread.sleep(Native Method) at org.apache.zookeeper.Login$1.run(Login.java:199) at java.lang.Thread.run(Thread.java:750) "IPC Client (32389254) connection to cod-1e3xsc06g8mjm-master0.exait3p.z1did1.b0.cloudera.site/10.154.114.171:8020 from hbase/cod-1e3xsc06g8mjm-worker124.exait3p.z1did1.b0.cloudera.site@EXAIT3P.Z1DID1.B0.CLOUDERA.SITE" #570386 daemon prio=5 os_prio=0 tid=0x00007fc02edf4800 nid=0x3b0b in Object.wait() [0x00007fc0c4844000] java.lang.Thread.State: TIMED_WAITING (on object monitor) at java.lang.Object.wait(Native Method) at org.apache.hadoop.ipc.Client$Connection.waitForWork(Client.java:1067) - locked <0x00007fc66871dcc0> (a org.apache.hadoop.ipc.Client$Connection) at org.apache.hadoop.ipc.Client$Connection.run(Client.java:1111) "ReadOnlyZKClient-cod-1e3xsc06g8mjm-master1.exait3p.z1did1.b0.cloudera.site:2181,cod-1e3xsc06g8mjm-master0.exait3p.z1did1.b0.cloudera.site:2181,cod-1e3xsc06g8mjm-leader0.exait3p.z1did1.b0.cloudera.site:2181@0x791016f3" #536959 daemon prio=5 os_prio=0 tid=0x00007fc09139b800 nid=0x2dee waiting on condition [0x00007fc03a6e5000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007fc4c2767578> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.DelayQueue.poll(DelayQueue.java:259) at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:323) at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$$Lambda$164/1053864765.run(Unknown Source) at java.lang.Thread.run(Thread.java:750) "AsyncConn Chore Service.Chore.1" #536958 daemon prio=5 os_prio=0 tid=0x00007fc03650b800 nid=0x2ded waiting on condition [0x00007fc03aceb000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007fc4c275f840> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1093) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:750) "ReadOnlyZKClient-cod-1e3xsc06g8mjm-master1.exait3p.z1did1.b0.cloudera.site:2181,cod-1e3xsc06g8mjm-master0.exait3p.z1did1.b0.cloudera.site:2181,cod-1e3xsc06g8mjm-leader0.exait3p.z1did1.b0.cloudera.site:2181@0x0154164a" #536858 daemon prio=5 os_prio=0 tid=0x00007fc11deda800 nid=0x2d6e waiting on condition [0x00007fc042ceb000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007fc4c275b578> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.DelayQueue.poll(DelayQueue.java:259) at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:323) at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$$Lambda$164/1053864765.run(Unknown Source) at java.lang.Thread.run(Thread.java:750) "AsyncConn Chore Service.Chore.1" #536857 daemon prio=5 os_prio=0 tid=0x00007fc11dd84800 nid=0x2d6d waiting on condition [0x00007fc042dec000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007fc4c2761578> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1093) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:750) "ReadOnlyZKClient-cod-1e3xsc06g8mjm-master1.exait3p.z1did1.b0.cloudera.site:2181,cod-1e3xsc06g8mjm-master0.exait3p.z1did1.b0.cloudera.site:2181,cod-1e3xsc06g8mjm-leader0.exait3p.z1did1.b0.cloudera.site:2181@0x4223b0aa" #536846 daemon prio=5 os_prio=0 tid=0x00007fc02e9ec000 nid=0x2d52 waiting on condition [0x00007fc0438f7000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007fc4c275faa0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.DelayQueue.poll(DelayQueue.java:259) at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:323) at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$$Lambda$164/1053864765.run(Unknown Source) at java.lang.Thread.run(Thread.java:750) "AsyncConn Chore Service.Chore.1" #536845 daemon prio=5 os_prio=0 tid=0x00007fc084216800 nid=0x2d51 waiting on condition [0x00007fc052fee000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007fc4c2767768> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1093) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:750) "ReadOnlyZKClient-cod-1e3xsc06g8mjm-master1.exait3p.z1did1.b0.cloudera.site:2181,cod-1e3xsc06g8mjm-master0.exait3p.z1did1.b0.cloudera.site:2181,cod-1e3xsc06g8mjm-leader0.exait3p.z1did1.b0.cloudera.site:2181@0x4c3f58e0" #536839 daemon prio=5 os_prio=0 tid=0x0000558b30700800 nid=0x2d4b waiting on condition [0x00007fc0483c2000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007fc4c275fcb0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.DelayQueue.poll(DelayQueue.java:259) at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:323) at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$$Lambda$164/1053864765.run(Unknown Source) at java.lang.Thread.run(Thread.java:750) "AsyncConn Chore Service.Chore.1" #536838 daemon prio=5 os_prio=0 tid=0x0000558b2196c000 nid=0x2d4a waiting on condition [0x00007fc0484c3000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007fc4c27679c8> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1093) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:750) "ReadOnlyZKClient-cod-1e3xsc06g8mjm-master1.exait3p.z1did1.b0.cloudera.site:2181,cod-1e3xsc06g8mjm-master0.exait3p.z1did1.b0.cloudera.site:2181,cod-1e3xsc06g8mjm-leader0.exait3p.z1did1.b0.cloudera.site:2181@0x28565e91" #536805 daemon prio=5 os_prio=0 tid=0x0000558b2fe94000 nid=0x2d20 waiting on condition [0x00007fc04a5e4000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007fc4c27617d8> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.DelayQueue.poll(DelayQueue.java:259) at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:323) at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$$Lambda$164/1053864765.run(Unknown Source) at java.lang.Thread.run(Thread.java:750) "AsyncConn Chore Service.Chore.1" #536804 daemon prio=5 os_prio=0 tid=0x0000558b21ace000 nid=0x2d1f waiting on condition [0x00007fc04a6e5000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007fc4c275fec0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1093) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:750) "ReadOnlyZKClient-cod-1e3xsc06g8mjm-master1.exait3p.z1did1.b0.cloudera.site:2181,cod-1e3xsc06g8mjm-master0.exait3p.z1did1.b0.cloudera.site:2181,cod-1e3xsc06g8mjm-leader0.exait3p.z1did1.b0.cloudera.site:2181@0x65168ee7" #536801 daemon prio=5 os_prio=0 tid=0x00007fc085063000 nid=0x2d1c waiting on condition [0x00007fc04a9e8000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007fc4c2767c08> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.DelayQueue.poll(DelayQueue.java:259) at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:323) at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$$Lambda$164/1053864765.run(Unknown Source) at java.lang.Thread.run(Thread.java:750) "AsyncConn Chore Service.Chore.1" #536800 daemon prio=5 os_prio=0 tid=0x00007fc093ad1800 nid=0x2d1b waiting on condition [0x00007fc04aae9000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007fc4c2760100> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1093) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:750) "ReadOnlyZKClient-cod-1e3xsc06g8mjm-master1.exait3p.z1did1.b0.cloudera.site:2181,cod-1e3xsc06g8mjm-master0.exait3p.z1did1.b0.cloudera.site:2181,cod-1e3xsc06g8mjm-leader0.exait3p.z1did1.b0.cloudera.site:2181@0x15145875" #536789 daemon prio=5 os_prio=0 tid=0x00007fc06d86c800 nid=0x2d0c waiting on condition [0x00007fc04b7f6000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007fc4c2767df8> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.DelayQueue.poll(DelayQueue.java:259) at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:323) at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$$Lambda$164/1053864765.run(Unknown Source) at java.lang.Thread.run(Thread.java:750) "AsyncConn Chore Service.Chore.1" #536788 daemon prio=5 os_prio=0 tid=0x00007fc03d4eb000 nid=0x2d0b waiting on condition [0x00007fc04b8f7000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007fc4c2767fe8> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1093) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:750) "ReadOnlyZKClient-cod-1e3xsc06g8mjm-master1.exait3p.z1did1.b0.cloudera.site:2181,cod-1e3xsc06g8mjm-master0.exait3p.z1did1.b0.cloudera.site:2181,cod-1e3xsc06g8mjm-leader0.exait3p.z1did1.b0.cloudera.site:2181@0x727ff19c" #536724 daemon prio=5 os_prio=0 tid=0x00007fc0b1610800 nid=0x2ca8 waiting on condition [0x00007fc0584c3000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007fc4c2760340> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.DelayQueue.poll(DelayQueue.java:259) at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:323) at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$$Lambda$164/1053864765.run(Unknown Source) at java.lang.Thread.run(Thread.java:750) "AsyncConn Chore Service.Chore.1" #536723 daemon prio=5 os_prio=0 tid=0x00007fc097b3f800 nid=0x2ca7 waiting on condition [0x00007fc0585c4000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007fc4c2768228> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1093) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:750) "ReadOnlyZKClient-cod-1e3xsc06g8mjm-master1.exait3p.z1did1.b0.cloudera.site:2181,cod-1e3xsc06g8mjm-master0.exait3p.z1did1.b0.cloudera.site:2181,cod-1e3xsc06g8mjm-leader0.exait3p.z1did1.b0.cloudera.site:2181@0x76296af3" #536708 daemon prio=5 os_prio=0 tid=0x00007fc0ae6ed800 nid=0x2c94 waiting on condition [0x00007fc0599d8000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007fc4c2760530> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.DelayQueue.poll(DelayQueue.java:259) at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:323) at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$$Lambda$164/1053864765.run(Unknown Source) at java.lang.Thread.run(Thread.java:750) "AsyncConn Chore Service.Chore.1" #536707 daemon prio=5 os_prio=0 tid=0x00007fc09ee53000 nid=0x2c93 waiting on condition [0x00007fc059ad9000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007fc4c2768468> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1093) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:750) "ReadOnlyZKClient-cod-1e3xsc06g8mjm-master1.exait3p.z1did1.b0.cloudera.site:2181,cod-1e3xsc06g8mjm-master0.exait3p.z1did1.b0.cloudera.site:2181,cod-1e3xsc06g8mjm-leader0.exait3p.z1did1.b0.cloudera.site:2181@0x62537836" #536699 daemon prio=5 os_prio=0 tid=0x00007fc05f6df000 nid=0x2c8b waiting on condition [0x00007fc05a4e3000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007fc4c2760720> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.DelayQueue.poll(DelayQueue.java:259) at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:323) at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$$Lambda$164/1053864765.run(Unknown Source) at java.lang.Thread.run(Thread.java:750) "AsyncConn Chore Service.Chore.1" #536698 daemon prio=5 os_prio=0 tid=0x00007fc05f86e000 nid=0x2c8a waiting on condition [0x00007fc05a6e5000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007fc4c27686a8> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1093) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:750) "ReadOnlyZKClient-cod-1e3xsc06g8mjm-master1.exait3p.z1did1.b0.cloudera.site:2181,cod-1e3xsc06g8mjm-master0.exait3p.z1did1.b0.cloudera.site:2181,cod-1e3xsc06g8mjm-leader0.exait3p.z1did1.b0.cloudera.site:2181@0x7142c0c5" #536583 daemon prio=5 os_prio=0 tid=0x00007fc037d87800 nid=0x2bcc waiting on condition [0x00007fc071ad9000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007fc4c2765578> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.DelayQueue.poll(DelayQueue.java:259) at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:323) at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$$Lambda$164/1053864765.run(Unknown Source) at java.lang.Thread.run(Thread.java:750) "AsyncConn Chore Service.Chore.1" #536582 daemon prio=5 os_prio=0 tid=0x00007fc092f50000 nid=0x2bcb waiting on condition [0x00007fc071bda000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007fc4c2760910> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1093) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:750) "ReadOnlyZKClient-cod-1e3xsc06g8mjm-master1.exait3p.z1did1.b0.cloudera.site:2181,cod-1e3xsc06g8mjm-master0.exait3p.z1did1.b0.cloudera.site:2181,cod-1e3xsc06g8mjm-leader0.exait3p.z1did1.b0.cloudera.site:2181@0x5e9c3338" #536570 daemon prio=5 os_prio=0 tid=0x0000558b30314000 nid=0x2bbf waiting on condition [0x00007fc0729e8000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007fc4c27688e8> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.DelayQueue.poll(DelayQueue.java:259) at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:323) at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$$Lambda$164/1053864765.run(Unknown Source) at java.lang.Thread.run(Thread.java:750) "AsyncConn Chore Service.Chore.1" #536569 daemon prio=5 os_prio=0 tid=0x0000558b30776000 nid=0x2bbe waiting on condition [0x00007fc072ae9000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007fc4c275b788> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1093) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:750) "ReadOnlyZKClient-cod-1e3xsc06g8mjm-master1.exait3p.z1did1.b0.cloudera.site:2181,cod-1e3xsc06g8mjm-master0.exait3p.z1did1.b0.cloudera.site:2181,cod-1e3xsc06g8mjm-leader0.exait3p.z1did1.b0.cloudera.site:2181@0x69b2369c" #536563 daemon prio=5 os_prio=0 tid=0x00007fc0b1be6800 nid=0x2bb8 waiting on condition [0x00007fc0730ef000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007fc4c2760b50> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.DelayQueue.poll(DelayQueue.java:259) at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:323) at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$$Lambda$164/1053864765.run(Unknown Source) at java.lang.Thread.run(Thread.java:750) "AsyncConn Chore Service.Chore.1" #536562 daemon prio=5 os_prio=0 tid=0x00007fc0b0ba9800 nid=0x2bb7 waiting on condition [0x00007fc0731f0000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007fc4c2765768> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1093) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:750) "ReadOnlyZKClient-cod-1e3xsc06g8mjm-master1.exait3p.z1did1.b0.cloudera.site:2181,cod-1e3xsc06g8mjm-master0.exait3p.z1did1.b0.cloudera.site:2181,cod-1e3xsc06g8mjm-leader0.exait3p.z1did1.b0.cloudera.site:2181@0x2cb6b356" #536548 daemon prio=5 os_prio=0 tid=0x00007fc03403b000 nid=0x2ba5 waiting on condition [0x00007fc073ffe000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007fc4c2760d40> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.DelayQueue.poll(DelayQueue.java:259) at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:323) at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$$Lambda$164/1053864765.run(Unknown Source) at java.lang.Thread.run(Thread.java:750) "AsyncConn Chore Service.Chore.1" #536547 daemon prio=5 os_prio=0 tid=0x00007fc046cae800 nid=0x2ba4 waiting on condition [0x00007fc0781c0000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007fc4c275b9c8> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1093) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:750) "ReadOnlyZKClient-cod-1e3xsc06g8mjm-master1.exait3p.z1did1.b0.cloudera.site:2181,cod-1e3xsc06g8mjm-master0.exait3p.z1did1.b0.cloudera.site:2181,cod-1e3xsc06g8mjm-leader0.exait3p.z1did1.b0.cloudera.site:2181@0x6f71f807" #536485 daemon prio=5 os_prio=0 tid=0x00007fc11ded8800 nid=0x2b1d waiting on condition [0x00007fc099cdb000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007fc4c27659a8> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.DelayQueue.poll(DelayQueue.java:259) at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:323) at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$$Lambda$164/1053864765.run(Unknown Source) at java.lang.Thread.run(Thread.java:750) "AsyncConn Chore Service.Chore.1" #536484 daemon prio=5 os_prio=0 tid=0x00007fc0a5cd9000 nid=0x2b1c waiting on condition [0x00007fc099ddc000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007fc4c275bc08> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1093) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:750) "ReadOnlyZKClient-cod-1e3xsc06g8mjm-master1.exait3p.z1did1.b0.cloudera.site:2181,cod-1e3xsc06g8mjm-master0.exait3p.z1did1.b0.cloudera.site:2181,cod-1e3xsc06g8mjm-leader0.exait3p.z1did1.b0.cloudera.site:2181@0x26938458" #536398 daemon prio=5 os_prio=0 tid=0x00007fc06cbaa000 nid=0x2a72 waiting on condition [0x00007fc0ba7e6000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007fc4c2765b98> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.DelayQueue.poll(DelayQueue.java:259) at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:323) at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$$Lambda$164/1053864765.run(Unknown Source) at java.lang.Thread.run(Thread.java:750) "AsyncConn Chore Service.Chore.1" #536397 daemon prio=5 os_prio=0 tid=0x00007fc06d866800 nid=0x2a71 waiting on condition [0x00007fc0ba9e8000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007fc4c275be48> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1093) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:750) "ReadOnlyZKClient-cod-1e3xsc06g8mjm-master1.exait3p.z1did1.b0.cloudera.site:2181,cod-1e3xsc06g8mjm-master0.exait3p.z1did1.b0.cloudera.site:2181,cod-1e3xsc06g8mjm-leader0.exait3p.z1did1.b0.cloudera.site:2181@0x01fa2426" #536392 daemon prio=5 os_prio=0 tid=0x00007fc0a52e0800 nid=0x2a6c waiting on condition [0x00007fc0bb2f1000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007fc4c2765d88> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.DelayQueue.poll(DelayQueue.java:259) at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:323) at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$$Lambda$164/1053864765.run(Unknown Source) at java.lang.Thread.run(Thread.java:750) "AsyncConn Chore Service.Chore.1" #536391 daemon prio=5 os_prio=0 tid=0x00007fc11f603000 nid=0x2a6b waiting on condition [0x00007fc0bb4f3000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007fc4c275c0a8> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1093) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:750) "ReadOnlyZKClient-cod-1e3xsc06g8mjm-master1.exait3p.z1did1.b0.cloudera.site:2181,cod-1e3xsc06g8mjm-master0.exait3p.z1did1.b0.cloudera.site:2181,cod-1e3xsc06g8mjm-leader0.exait3p.z1did1.b0.cloudera.site:2181@0x362fcbf5" #536385 daemon prio=5 os_prio=0 tid=0x00007fc095f8e800 nid=0x2a65 waiting on condition [0x00007fc0bbcfb000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007fc4c2757578> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.DelayQueue.poll(DelayQueue.java:259) at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:323) at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$$Lambda$164/1053864765.run(Unknown Source) at java.lang.Thread.run(Thread.java:750) "AsyncConn Chore Service.Chore.1" #536384 daemon prio=5 os_prio=0 tid=0x00007fc0b1494000 nid=0x2a64 waiting on condition [0x00007fc0bbdfc000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007fc4c2765f78> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1093) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:750) "ReadOnlyZKClient-cod-1e3xsc06g8mjm-master1.exait3p.z1did1.b0.cloudera.site:2181,cod-1e3xsc06g8mjm-master0.exait3p.z1did1.b0.cloudera.site:2181,cod-1e3xsc06g8mjm-leader0.exait3p.z1did1.b0.cloudera.site:2181@0x6c160b94" #536330 daemon prio=5 os_prio=0 tid=0x00007fc11df2b000 nid=0x2a1b waiting on condition [0x00007fc0cca17000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007fc4c275c308> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.DelayQueue.poll(DelayQueue.java:259) at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:323) at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$$Lambda$164/1053864765.run(Unknown Source) at java.lang.Thread.run(Thread.java:750) "AsyncConn Chore Service.Chore.1" #536329 daemon prio=5 os_prio=0 tid=0x00007fc0a5256800 nid=0x2a1a waiting on condition [0x00007fc0ccd1a000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007fc4c2757768> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1093) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:750) "ReadOnlyZKClient-cod-1e3xsc06g8mjm-master1.exait3p.z1did1.b0.cloudera.site:2181,cod-1e3xsc06g8mjm-master0.exait3p.z1did1.b0.cloudera.site:2181,cod-1e3xsc06g8mjm-leader0.exait3p.z1did1.b0.cloudera.site:2181@0x5d19826e" #535643 daemon prio=5 os_prio=0 tid=0x0000558b13ea9000 nid=0x1929 waiting on condition [0x00007fc0a15d4000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007fc6a623c978> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.DelayQueue.poll(DelayQueue.java:259) at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:323) at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$$Lambda$164/1053864765.run(Unknown Source) at java.lang.Thread.run(Thread.java:750) "AsyncConn Chore Service.Chore.1" #535642 daemon prio=5 os_prio=0 tid=0x0000558b2e626000 nid=0x1928 waiting on condition [0x00007fc0a2bea000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007fc6a6230bd0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1093) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:750) "ReadOnlyZKClient-cod-1e3xsc06g8mjm-master1.exait3p.z1did1.b0.cloudera.site:2181,cod-1e3xsc06g8mjm-master0.exait3p.z1did1.b0.cloudera.site:2181,cod-1e3xsc06g8mjm-leader0.exait3p.z1did1.b0.cloudera.site:2181@0x46367972" #535637 daemon prio=5 os_prio=0 tid=0x00007fc02e93d000 nid=0x191a waiting on condition [0x00007fc0c1102000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007fc6a6234978> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.DelayQueue.poll(DelayQueue.java:259) at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:323) at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$$Lambda$164/1053864765.run(Unknown Source) at java.lang.Thread.run(Thread.java:750) "AsyncConn Chore Service.Chore.1" #535636 daemon prio=5 os_prio=0 tid=0x00007fc034114800 nid=0x1919 waiting on condition [0x00007fc0c2112000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007fc6a623cb68> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1093) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:750) "ReadOnlyZKClient-cod-1e3xsc06g8mjm-master1.exait3p.z1did1.b0.cloudera.site:2181,cod-1e3xsc06g8mjm-master0.exait3p.z1did1.b0.cloudera.site:2181,cod-1e3xsc06g8mjm-leader0.exait3p.z1did1.b0.cloudera.site:2181@0x00f7e1ec" #535616 daemon prio=5 os_prio=0 tid=0x0000558b215b3800 nid=0x18df waiting on condition [0x00007fc0e04c7000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007fc6a6230e10> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.DelayQueue.poll(DelayQueue.java:259) at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:323) at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$$Lambda$164/1053864765.run(Unknown Source) at java.lang.Thread.run(Thread.java:750) "AsyncConn Chore Service.Chore.1" #535615 daemon prio=5 os_prio=0 tid=0x0000558b1ebeb800 nid=0x18de waiting on condition [0x00007fc02adec000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007fc6a6234b68> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1093) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:750) "ReadOnlyZKClient-cod-1e3xsc06g8mjm-master1.exait3p.z1did1.b0.cloudera.site:2181,cod-1e3xsc06g8mjm-master0.exait3p.z1did1.b0.cloudera.site:2181,cod-1e3xsc06g8mjm-leader0.exait3p.z1did1.b0.cloudera.site:2181@0x51d6aef4" #535575 daemon prio=5 os_prio=0 tid=0x00007fc0b5caf800 nid=0x184f waiting on condition [0x00007fc0a3efd000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007fc50d0c3f48> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.DelayQueue.poll(DelayQueue.java:259) at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:323) at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$$Lambda$164/1053864765.run(Unknown Source) at java.lang.Thread.run(Thread.java:750) "AsyncConn Chore Service.Chore.1" #535574 daemon prio=5 os_prio=0 tid=0x00007fc03cbf6000 nid=0x184e waiting on condition [0x00007fc0a3ffe000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007fc50d0d3d78> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1093) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:750) "ReadOnlyZKClient-cod-1e3xsc06g8mjm-master1.exait3p.z1did1.b0.cloudera.site:2181,cod-1e3xsc06g8mjm-master0.exait3p.z1did1.b0.cloudera.site:2181,cod-1e3xsc06g8mjm-leader0.exait3p.z1did1.b0.cloudera.site:2181@0x6245171b" #535558 daemon prio=5 os_prio=0 tid=0x00007fc02ddcc800 nid=0x1837 waiting on condition [0x00007fc0b99d8000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007fc50d0c4158> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.DelayQueue.poll(DelayQueue.java:259) at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:323) at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$$Lambda$164/1053864765.run(Unknown Source) at java.lang.Thread.run(Thread.java:750) "AsyncConn Chore Service.Chore.1" #535557 daemon prio=5 os_prio=0 tid=0x00007fc092f54800 nid=0x1836 waiting on condition [0x00007fc0b9ddc000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007fc50d0d3fd8> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1093) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:750) "ReadOnlyZKClient-cod-1e3xsc06g8mjm-master1.exait3p.z1did1.b0.cloudera.site:2181,cod-1e3xsc06g8mjm-master0.exait3p.z1did1.b0.cloudera.site:2181,cod-1e3xsc06g8mjm-leader0.exait3p.z1did1.b0.cloudera.site:2181@0x52aafd55" #535524 daemon prio=5 os_prio=0 tid=0x00007fc07c59e800 nid=0x17ef waiting on condition [0x00007fc0c08fa000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007fc50d0c4368> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.DelayQueue.poll(DelayQueue.java:259) at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:323) at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$$Lambda$164/1053864765.run(Unknown Source) at java.lang.Thread.run(Thread.java:750) "AsyncConn Chore Service.Chore.1" #535523 daemon prio=5 os_prio=0 tid=0x00007fc07c1e4800 nid=0x17ee waiting on condition [0x00007fc0c0afc000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007fc50d0d4238> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1093) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:750) "ReadOnlyZKClient-cod-1e3xsc06g8mjm-master1.exait3p.z1did1.b0.cloudera.site:2181,cod-1e3xsc06g8mjm-master0.exait3p.z1did1.b0.cloudera.site:2181,cod-1e3xsc06g8mjm-leader0.exait3p.z1did1.b0.cloudera.site:2181@0x6b5d710d" #535517 daemon prio=5 os_prio=0 tid=0x00007fc02e642800 nid=0x17e3 waiting on condition [0x00007fc0c1607000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007fc50d0c4558> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.DelayQueue.poll(DelayQueue.java:259) at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:323) at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$$Lambda$164/1053864765.run(Unknown Source) at java.lang.Thread.run(Thread.java:750) "AsyncConn Chore Service.Chore.1" #535516 daemon prio=5 os_prio=0 tid=0x00007fc02cf15000 nid=0x17e2 waiting on condition [0x00007fc0c1809000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007fc50d0d4478> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1093) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:750) "ReadOnlyZKClient-cod-1e3xsc06g8mjm-master1.exait3p.z1did1.b0.cloudera.site:2181,cod-1e3xsc06g8mjm-master0.exait3p.z1did1.b0.cloudera.site:2181,cod-1e3xsc06g8mjm-leader0.exait3p.z1did1.b0.cloudera.site:2181@0x0ab47275" #535513 daemon prio=5 os_prio=0 tid=0x00007fc0a625d800 nid=0x17df waiting on condition [0x00007fc0c1b0c000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007fc50d0c4748> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.DelayQueue.poll(DelayQueue.java:259) at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:323) at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$$Lambda$164/1053864765.run(Unknown Source) at java.lang.Thread.run(Thread.java:750) "AsyncConn Chore Service.Chore.1" #535512 daemon prio=5 os_prio=0 tid=0x00007fc06c280000 nid=0x17de waiting on condition [0x00007fc0c2516000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007fc50d0d46b8> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1093) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:750) "ReadOnlyZKClient-cod-1e3xsc06g8mjm-master1.exait3p.z1did1.b0.cloudera.site:2181,cod-1e3xsc06g8mjm-master0.exait3p.z1did1.b0.cloudera.site:2181,cod-1e3xsc06g8mjm-leader0.exait3p.z1did1.b0.cloudera.site:2181@0x4c50c43e" #535497 daemon prio=5 os_prio=0 tid=0x00007fc0aaf34800 nid=0x17c1 waiting on condition [0x00007fc0c3c2d000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007fc50d0cfd78> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.DelayQueue.poll(DelayQueue.java:259) at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:323) at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$$Lambda$164/1053864765.run(Unknown Source) at java.lang.Thread.run(Thread.java:750) "AsyncConn Chore Service.Chore.1" #535496 daemon prio=5 os_prio=0 tid=0x00007fc0bc509000 nid=0x17c0 waiting on condition [0x00007fc0c3f30000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007fc50d0c4938> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1093) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:750) "ReadOnlyZKClient-cod-1e3xsc06g8mjm-master1.exait3p.z1did1.b0.cloudera.site:2181,cod-1e3xsc06g8mjm-master0.exait3p.z1did1.b0.cloudera.site:2181,cod-1e3xsc06g8mjm-leader0.exait3p.z1did1.b0.cloudera.site:2181@0x51c4a8b3" #535462 daemon prio=5 os_prio=0 tid=0x0000558b21fa9000 nid=0x1754 waiting on condition [0x00007fc0c77f6000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007fc5bf245b78> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.DelayQueue.poll(DelayQueue.java:259) at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:323) at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$$Lambda$164/1053864765.run(Unknown Source) at java.lang.Thread.run(Thread.java:750) "AsyncConn Chore Service.Chore.1" #535461 daemon prio=5 os_prio=0 tid=0x0000558b23d6e000 nid=0x1753 waiting on condition [0x00007fc0cc28b000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007fc5bf241be0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1093) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:750) "ReadOnlyZKClient-cod-1e3xsc06g8mjm-master1.exait3p.z1did1.b0.cloudera.site:2181,cod-1e3xsc06g8mjm-master0.exait3p.z1did1.b0.cloudera.site:2181,cod-1e3xsc06g8mjm-leader0.exait3p.z1did1.b0.cloudera.site:2181@0x7fd2ff1e" #535455 daemon prio=5 os_prio=0 tid=0x00007fc07e03c000 nid=0x174c waiting on condition [0x00007fc0cdfe0000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007fc5bf245d68> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.DelayQueue.poll(DelayQueue.java:259) at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:323) at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$$Lambda$164/1053864765.run(Unknown Source) at java.lang.Thread.run(Thread.java:750) "AsyncConn Chore Service.Chore.1" #535454 daemon prio=5 os_prio=0 tid=0x00007fc0c8378000 nid=0x174b waiting on condition [0x00007fc0cf5f6000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007fc5bf243b78> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1093) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:750) "ReadOnlyZKClient-cod-1e3xsc06g8mjm-master1.exait3p.z1did1.b0.cloudera.site:2181,cod-1e3xsc06g8mjm-master0.exait3p.z1did1.b0.cloudera.site:2181,cod-1e3xsc06g8mjm-leader0.exait3p.z1did1.b0.cloudera.site:2181@0x27850096" #535432 daemon prio=5 os_prio=0 tid=0x00007fc0ad4cf800 nid=0x1712 waiting on condition [0x00007fc0a14d3000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007fc5bf241e20> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.DelayQueue.poll(DelayQueue.java:259) at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:323) at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$$Lambda$164/1053864765.run(Unknown Source) at java.lang.Thread.run(Thread.java:750) "AsyncConn Chore Service.Chore.1" #535431 daemon prio=5 os_prio=0 tid=0x00007fc02e188800 nid=0x1711 waiting on condition [0x00007fc043bfa000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007fc5bf245f58> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1093) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:750) "ReadOnlyZKClient-cod-1e3xsc06g8mjm-master1.exait3p.z1did1.b0.cloudera.site:2181,cod-1e3xsc06g8mjm-master0.exait3p.z1did1.b0.cloudera.site:2181,cod-1e3xsc06g8mjm-leader0.exait3p.z1did1.b0.cloudera.site:2181@0x1bd46007" #535402 daemon prio=5 os_prio=0 tid=0x00007fc11d45f800 nid=0x16de waiting on condition [0x00007fc078ecd000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007fc4c2757a88> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.DelayQueue.poll(DelayQueue.java:259) at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:323) at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$$Lambda$164/1053864765.run(Unknown Source) at java.lang.Thread.run(Thread.java:750) "AsyncConn Chore Service.Chore.1" #535401 daemon prio=5 os_prio=0 tid=0x00007fc11fbaf800 nid=0x16dd waiting on condition [0x00007fc0794d3000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007fc4c275c538> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1093) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:750) "ReadOnlyZKClient-cod-1e3xsc06g8mjm-master1.exait3p.z1did1.b0.cloudera.site:2181,cod-1e3xsc06g8mjm-master0.exait3p.z1did1.b0.cloudera.site:2181,cod-1e3xsc06g8mjm-leader0.exait3p.z1did1.b0.cloudera.site:2181@0x0bc28794" #535396 daemon prio=5 os_prio=0 tid=0x00007fc02d307800 nid=0x16d8 waiting on condition [0x00007fc0c01f3000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007fc4c2760f30> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.DelayQueue.poll(DelayQueue.java:259) at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:323) at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$$Lambda$164/1053864765.run(Unknown Source) at java.lang.Thread.run(Thread.java:750) "AsyncConn Chore Service.Chore.1" #535395 daemon prio=5 os_prio=0 tid=0x00007fc02dcd9800 nid=0x16d7 waiting on condition [0x00007fc0c67e6000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007fc4c2757c78> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1093) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:750) "ReadOnlyZKClient-cod-1e3xsc06g8mjm-master1.exait3p.z1did1.b0.cloudera.site:2181,cod-1e3xsc06g8mjm-master0.exait3p.z1did1.b0.cloudera.site:2181,cod-1e3xsc06g8mjm-leader0.exait3p.z1did1.b0.cloudera.site:2181@0x5872589a" #535390 daemon prio=5 os_prio=0 tid=0x00007fc99acd7800 nid=0x16b9 waiting on condition [0x00007fc0bafee000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007fc4c275c778> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.DelayQueue.poll(DelayQueue.java:259) at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:323) at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$$Lambda$164/1053864765.run(Unknown Source) at java.lang.Thread.run(Thread.java:750) "AsyncConn Chore Service.Chore.1" #535389 daemon prio=5 os_prio=0 tid=0x00007fc0a6b97000 nid=0x16b8 waiting on condition [0x00007fc0c09fb000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007fc4c2757eb8> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1093) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:750) "ReadOnlyZKClient-cod-1e3xsc06g8mjm-master1.exait3p.z1did1.b0.cloudera.site:2181,cod-1e3xsc06g8mjm-master0.exait3p.z1did1.b0.cloudera.site:2181,cod-1e3xsc06g8mjm-leader0.exait3p.z1did1.b0.cloudera.site:2181@0x49328ac1" #535223 daemon prio=5 os_prio=0 tid=0x0000558b14465000 nid=0x1276 waiting on condition [0x00007fc0605c4000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007fc4c2768ad8> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.DelayQueue.poll(DelayQueue.java:259) at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:323) at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$$Lambda$164/1053864765.run(Unknown Source) at java.lang.Thread.run(Thread.java:750) "AsyncConn Chore Service.Chore.1" #535222 daemon prio=5 os_prio=0 tid=0x0000558b2e0e9000 nid=0x1275 waiting on condition [0x00007fc0607c6000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007fc4c2768ce8> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1093) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:750) "ReadOnlyZKClient-cod-1e3xsc06g8mjm-master1.exait3p.z1did1.b0.cloudera.site:2181,cod-1e3xsc06g8mjm-master0.exait3p.z1did1.b0.cloudera.site:2181,cod-1e3xsc06g8mjm-leader0.exait3p.z1did1.b0.cloudera.site:2181@0x789fcda1" #535095 daemon prio=5 os_prio=0 tid=0x0000558b308fc800 nid=0x117b waiting on condition [0x00007fc0292d1000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007fc4fa8e6578> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.DelayQueue.poll(DelayQueue.java:259) at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:323) at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$$Lambda$164/1053864765.run(Unknown Source) at java.lang.Thread.run(Thread.java:750) "AsyncConn Chore Service.Chore.1" #535094 daemon prio=5 os_prio=0 tid=0x0000558b2a8a8800 nid=0x117a waiting on condition [0x00007fc0293d2000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007fc4fa8e6768> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1093) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:750) "ReadOnlyZKClient-cod-1e3xsc06g8mjm-master1.exait3p.z1did1.b0.cloudera.site:2181,cod-1e3xsc06g8mjm-master0.exait3p.z1did1.b0.cloudera.site:2181,cod-1e3xsc06g8mjm-leader0.exait3p.z1did1.b0.cloudera.site:2181@0x4677946f" #535090 daemon prio=5 os_prio=0 tid=0x00007fc02ccfa000 nid=0x1176 waiting on condition [0x00007fc0297d6000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007fc4fa8e8758> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.DelayQueue.poll(DelayQueue.java:259) at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:323) at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$$Lambda$164/1053864765.run(Unknown Source) at java.lang.Thread.run(Thread.java:750) "AsyncConn Chore Service.Chore.1" #535089 daemon prio=5 os_prio=0 tid=0x00007fc047562000 nid=0x1175 waiting on condition [0x00007fc0298d7000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007fc4fa8e8948> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1093) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:750) "ReadOnlyZKClient-cod-1e3xsc06g8mjm-master1.exait3p.z1did1.b0.cloudera.site:2181,cod-1e3xsc06g8mjm-master0.exait3p.z1did1.b0.cloudera.site:2181,cod-1e3xsc06g8mjm-leader0.exait3p.z1did1.b0.cloudera.site:2181@0x2b715fc4" #535084 daemon prio=5 os_prio=0 tid=0x00007fc06f9df800 nid=0x116f waiting on condition [0x00007fc029ddc000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007fc4fa8ea578> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.DelayQueue.poll(DelayQueue.java:259) at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:323) at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$$Lambda$164/1053864765.run(Unknown Source) at java.lang.Thread.run(Thread.java:750) "AsyncConn Chore Service.Chore.1" #535083 daemon prio=5 os_prio=0 tid=0x00007fc11db2e000 nid=0x116e waiting on condition [0x00007fc029edd000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007fc4fa8e69a8> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1093) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:750) "ReadOnlyZKClient-cod-1e3xsc06g8mjm-master1.exait3p.z1did1.b0.cloudera.site:2181,cod-1e3xsc06g8mjm-master0.exait3p.z1did1.b0.cloudera.site:2181,cod-1e3xsc06g8mjm-leader0.exait3p.z1did1.b0.cloudera.site:2181@0x0b9fa07f" #535065 daemon prio=5 os_prio=0 tid=0x00007fc037dfe000 nid=0x1145 waiting on condition [0x00007fc02afee000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007fc50d0c4b78> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.DelayQueue.poll(DelayQueue.java:259) at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:323) at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$$Lambda$164/1053864765.run(Unknown Source) at java.lang.Thread.run(Thread.java:750) "AsyncConn Chore Service.Chore.1" #535064 daemon prio=5 os_prio=0 tid=0x00007fc02cbee000 nid=0x1144 waiting on condition [0x00007fc02b2f1000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007fc50d0d48f8> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1093) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:750) "ReadOnlyZKClient-cod-1e3xsc06g8mjm-master1.exait3p.z1did1.b0.cloudera.site:2181,cod-1e3xsc06g8mjm-master0.exait3p.z1did1.b0.cloudera.site:2181,cod-1e3xsc06g8mjm-leader0.exait3p.z1did1.b0.cloudera.site:2181@0x029123cd" #535053 daemon prio=5 os_prio=0 tid=0x0000558b23d6d800 nid=0x112d waiting on condition [0x00007fc02b9f8000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007fc50d0c4d68> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.DelayQueue.poll(DelayQueue.java:259) at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:323) at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$$Lambda$164/1053864765.run(Unknown Source) at java.lang.Thread.run(Thread.java:750) "AsyncConn Chore Service.Chore.1" #535052 daemon prio=5 os_prio=0 tid=0x0000558b307ae000 nid=0x112c waiting on condition [0x00007fc02baf9000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007fc50d0c61a8> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1093) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:750) "ReadOnlyZKClient-cod-1e3xsc06g8mjm-master1.exait3p.z1did1.b0.cloudera.site:2181,cod-1e3xsc06g8mjm-master0.exait3p.z1did1.b0.cloudera.site:2181,cod-1e3xsc06g8mjm-leader0.exait3p.z1did1.b0.cloudera.site:2181@0x1541be11" #535031 daemon prio=5 os_prio=0 tid=0x0000558b2e2da800 nid=0x110a waiting on condition [0x00007fc030ac9000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007fc4c2768f48> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.DelayQueue.poll(DelayQueue.java:259) at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:323) at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$$Lambda$164/1053864765.run(Unknown Source) at java.lang.Thread.run(Thread.java:750) "AsyncConn Chore Service.Chore.1" #535030 daemon prio=5 os_prio=0 tid=0x0000558b2646c000 nid=0x1109 waiting on condition [0x00007fc030bca000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007fc4c2769138> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1093) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:750) "ReadOnlyZKClient-cod-1e3xsc06g8mjm-master1.exait3p.z1did1.b0.cloudera.site:2181,cod-1e3xsc06g8mjm-master0.exait3p.z1did1.b0.cloudera.site:2181,cod-1e3xsc06g8mjm-leader0.exait3p.z1did1.b0.cloudera.site:2181@0x1a426dfb" #534995 daemon prio=5 os_prio=0 tid=0x00007fc0a6a2c800 nid=0x10dc waiting on condition [0x00007fc09aceb000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007fc4c275c968> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.DelayQueue.poll(DelayQueue.java:259) at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:323) at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$$Lambda$164/1053864765.run(Unknown Source) at java.lang.Thread.run(Thread.java:750) "AsyncConn Chore Service.Chore.1" #534994 daemon prio=5 os_prio=0 tid=0x00007fc11dd41800 nid=0x10db waiting on condition [0x00007fc0a08c7000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007fc4c27580f8> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1093) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:750) "ReadOnlyZKClient-cod-1e3xsc06g8mjm-master1.exait3p.z1did1.b0.cloudera.site:2181,cod-1e3xsc06g8mjm-master0.exait3p.z1did1.b0.cloudera.site:2181,cod-1e3xsc06g8mjm-leader0.exait3p.z1did1.b0.cloudera.site:2181@0x56008ecc" #534972 daemon prio=5 os_prio=0 tid=0x00007fc0b4bb7000 nid=0x10c1 waiting on condition [0x00007fc09aeed000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007fc4c2769378> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.DelayQueue.poll(DelayQueue.java:259) at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:323) at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$$Lambda$164/1053864765.run(Unknown Source) at java.lang.Thread.run(Thread.java:750) "AsyncConn Chore Service.Chore.1" #534971 daemon prio=5 os_prio=0 tid=0x00007fc06f951000 nid=0x10c0 waiting on condition [0x00007fc09adec000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007fc4c275cb58> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1093) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:750) "ReadOnlyZKClient-cod-1e3xsc06g8mjm-master1.exait3p.z1did1.b0.cloudera.site:2181,cod-1e3xsc06g8mjm-master0.exait3p.z1did1.b0.cloudera.site:2181,cod-1e3xsc06g8mjm-leader0.exait3p.z1did1.b0.cloudera.site:2181@0x0104fd74" #534965 daemon prio=5 os_prio=0 tid=0x00007fc0aedb2800 nid=0x10a6 waiting on condition [0x00007fc09b3f2000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007fc4c2758338> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.DelayQueue.poll(DelayQueue.java:259) at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:323) at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$$Lambda$164/1053864765.run(Unknown Source) at java.lang.Thread.run(Thread.java:750) "AsyncConn Chore Service.Chore.1" #534964 daemon prio=5 os_prio=0 tid=0x00007fc09c8fe000 nid=0x10a5 waiting on condition [0x00007fc09b4f3000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007fc4c275cdb8> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1093) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:750) "ReadOnlyZKClient-cod-1e3xsc06g8mjm-master1.exait3p.z1did1.b0.cloudera.site:2181,cod-1e3xsc06g8mjm-master0.exait3p.z1did1.b0.cloudera.site:2181,cod-1e3xsc06g8mjm-leader0.exait3p.z1did1.b0.cloudera.site:2181@0x7ee0d95c" #534959 daemon prio=5 os_prio=0 tid=0x0000558b30702000 nid=0x109f waiting on condition [0x00007fc09befd000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007fc4c2758528> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.DelayQueue.poll(DelayQueue.java:259) at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:323) at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$$Lambda$164/1053864765.run(Unknown Source) at java.lang.Thread.run(Thread.java:750) "AsyncConn Chore Service.Chore.1" #534958 daemon prio=5 os_prio=0 tid=0x0000558b1f249000 nid=0x109e waiting on condition [0x00007fc09bdfc000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007fc4c275d018> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1093) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:750) "ReadOnlyZKClient-cod-1e3xsc06g8mjm-master1.exait3p.z1did1.b0.cloudera.site:2181,cod-1e3xsc06g8mjm-master0.exait3p.z1did1.b0.cloudera.site:2181,cod-1e3xsc06g8mjm-leader0.exait3p.z1did1.b0.cloudera.site:2181@0x220432e1" #534938 daemon prio=5 os_prio=0 tid=0x00007fc08cc1a000 nid=0x1088 waiting on condition [0x00007fc0a11d0000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007fc4c2758718> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.DelayQueue.poll(DelayQueue.java:259) at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:323) at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$$Lambda$164/1053864765.run(Unknown Source) at java.lang.Thread.run(Thread.java:750) "AsyncConn Chore Service.Chore.1" #534937 daemon prio=5 os_prio=0 tid=0x00007fc07d6b9800 nid=0x1087 waiting on condition [0x00007fc0ba1e0000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007fc4c275d278> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1093) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:750) "ReadOnlyZKClient-cod-1e3xsc06g8mjm-master1.exait3p.z1did1.b0.cloudera.site:2181,cod-1e3xsc06g8mjm-master0.exait3p.z1did1.b0.cloudera.site:2181,cod-1e3xsc06g8mjm-leader0.exait3p.z1did1.b0.cloudera.site:2181@0x5c2116da" #534934 daemon prio=5 os_prio=0 tid=0x0000558b1e2d6800 nid=0x1084 waiting on condition [0x00007fc0a21e0000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007fc4c2758908> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.DelayQueue.poll(DelayQueue.java:259) at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:323) at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$$Lambda$164/1053864765.run(Unknown Source) at java.lang.Thread.run(Thread.java:750) "AsyncConn Chore Service.Chore.1" #534933 daemon prio=5 os_prio=0 tid=0x0000558b20f3d000 nid=0x1083 waiting on condition [0x00007fc0a20df000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007fc4c275d578> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1093) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:750) "ReadOnlyZKClient-cod-1e3xsc06g8mjm-master1.exait3p.z1did1.b0.cloudera.site:2181,cod-1e3xsc06g8mjm-master0.exait3p.z1did1.b0.cloudera.site:2181,cod-1e3xsc06g8mjm-leader0.exait3p.z1did1.b0.cloudera.site:2181@0x786ee953" #534928 daemon prio=5 os_prio=0 tid=0x00007fc0467ec800 nid=0x107d waiting on condition [0x00007fc0a0dcc000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007fc4c2758af8> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.DelayQueue.poll(DelayQueue.java:259) at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:323) at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$$Lambda$164/1053864765.run(Unknown Source) at java.lang.Thread.run(Thread.java:750) "AsyncConn Chore Service.Chore.1" #534927 daemon prio=5 os_prio=0 tid=0x00007fc076bf8800 nid=0x107c waiting on condition [0x00007fc0bb1f0000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007fc4c275d7d8> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1093) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:750) "ReadOnlyZKClient-cod-1e3xsc06g8mjm-master1.exait3p.z1did1.b0.cloudera.site:2181,cod-1e3xsc06g8mjm-master0.exait3p.z1did1.b0.cloudera.site:2181,cod-1e3xsc06g8mjm-leader0.exait3p.z1did1.b0.cloudera.site:2181@0x2e97d327" #534878 daemon prio=5 os_prio=0 tid=0x00007fc0b5a25800 nid=0x1024 waiting on condition [0x00007fc0322e1000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007fc4c2758ce8> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.DelayQueue.poll(DelayQueue.java:259) at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:323) at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$$Lambda$164/1053864765.run(Unknown Source) at java.lang.Thread.run(Thread.java:750) "AsyncConn Chore Service.Chore.1" #534877 daemon prio=5 os_prio=0 tid=0x00007fc0b5a24800 nid=0x1023 waiting on condition [0x00007fc0323e2000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007fc4c275da38> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1093) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:750) "ReadOnlyZKClient-cod-1e3xsc06g8mjm-master1.exait3p.z1did1.b0.cloudera.site:2181,cod-1e3xsc06g8mjm-master0.exait3p.z1did1.b0.cloudera.site:2181,cod-1e3xsc06g8mjm-leader0.exait3p.z1did1.b0.cloudera.site:2181@0x06df12e8" #534869 daemon prio=5 os_prio=0 tid=0x00007fc037bb1800 nid=0x101b waiting on condition [0x00007fc0329e8000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007fc4c2758ed8> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.DelayQueue.poll(DelayQueue.java:259) at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:323) at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$$Lambda$164/1053864765.run(Unknown Source) at java.lang.Thread.run(Thread.java:750) "AsyncConn Chore Service.Chore.1" #534868 daemon prio=5 os_prio=0 tid=0x00007fc037bb1000 nid=0x101a waiting on condition [0x00007fc032ae9000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007fc4c275dc78> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1093) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:750) "ReadOnlyZKClient-cod-1e3xsc06g8mjm-master1.exait3p.z1did1.b0.cloudera.site:2181,cod-1e3xsc06g8mjm-master0.exait3p.z1did1.b0.cloudera.site:2181,cod-1e3xsc06g8mjm-leader0.exait3p.z1did1.b0.cloudera.site:2181@0x454801fe" #534833 daemon prio=5 os_prio=0 tid=0x00007fc05c6aa800 nid=0xfe1 waiting on condition [0x00007fc0382c1000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007fc4c27590c8> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.DelayQueue.poll(DelayQueue.java:259) at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:323) at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$$Lambda$164/1053864765.run(Unknown Source) at java.lang.Thread.run(Thread.java:750) "AsyncConn Chore Service.Chore.1" #534832 daemon prio=5 os_prio=0 tid=0x00007fc0a5479000 nid=0xfe0 waiting on condition [0x00007fc0383c2000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007fc4c275deb8> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1093) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:750) "ReadOnlyZKClient-cod-1e3xsc06g8mjm-master1.exait3p.z1did1.b0.cloudera.site:2181,cod-1e3xsc06g8mjm-master0.exait3p.z1did1.b0.cloudera.site:2181,cod-1e3xsc06g8mjm-leader0.exait3p.z1did1.b0.cloudera.site:2181@0x6095f2a9" #534812 daemon prio=5 os_prio=0 tid=0x0000558b30252800 nid=0xfc9 waiting on condition [0x00007fc0390cf000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007fc5c9cc0d98> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.DelayQueue.poll(DelayQueue.java:259) at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:323) at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$$Lambda$164/1053864765.run(Unknown Source) at java.lang.Thread.run(Thread.java:750) "AsyncConn Chore Service.Chore.1" #534811 daemon prio=5 os_prio=0 tid=0x0000558b30252000 nid=0xfc8 waiting on condition [0x00007fc0391d0000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007fc5c9cc2db8> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1093) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:750) "ReadOnlyZKClient-cod-1e3xsc06g8mjm-master1.exait3p.z1did1.b0.cloudera.site:2181,cod-1e3xsc06g8mjm-master0.exait3p.z1did1.b0.cloudera.site:2181,cod-1e3xsc06g8mjm-leader0.exait3p.z1did1.b0.cloudera.site:2181@0x2649bbb5" #534806 daemon prio=5 os_prio=0 tid=0x0000558b2772c000 nid=0xfc3 waiting on condition [0x00007fc039fde000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007fc5c9cc0f88> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.DelayQueue.poll(DelayQueue.java:259) at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:323) at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$$Lambda$164/1053864765.run(Unknown Source) at java.lang.Thread.run(Thread.java:750) "AsyncConn Chore Service.Chore.1" #534805 daemon prio=5 os_prio=0 tid=0x0000558b2772a800 nid=0xfc2 waiting on condition [0x00007fc03a0df000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007fc5c9cc2ff8> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1093) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:750) "ReadOnlyZKClient-cod-1e3xsc06g8mjm-master1.exait3p.z1did1.b0.cloudera.site:2181,cod-1e3xsc06g8mjm-master0.exait3p.z1did1.b0.cloudera.site:2181,cod-1e3xsc06g8mjm-leader0.exait3p.z1did1.b0.cloudera.site:2181@0x280ed002" #534778 daemon prio=5 os_prio=0 tid=0x00007fc0a61d8800 nid=0xfa4 waiting on condition [0x00007fc03a9e8000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007fc5c9cc1178> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.DelayQueue.poll(DelayQueue.java:259) at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:323) at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$$Lambda$164/1053864765.run(Unknown Source) at java.lang.Thread.run(Thread.java:750) "AsyncConn Chore Service.Chore.1" #534777 daemon prio=5 os_prio=0 tid=0x00007fc0b59c8000 nid=0xfa3 waiting on condition [0x00007fc03aae9000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007fc5c9cc1388> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1093) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:750) "ReadOnlyZKClient-cod-1e3xsc06g8mjm-master1.exait3p.z1did1.b0.cloudera.site:2181,cod-1e3xsc06g8mjm-master0.exait3p.z1did1.b0.cloudera.site:2181,cod-1e3xsc06g8mjm-leader0.exait3p.z1did1.b0.cloudera.site:2181@0x7629f96a" #534771 daemon prio=5 os_prio=0 tid=0x00007fc08d5aa800 nid=0xf9c waiting on condition [0x00007fc03afee000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007fc5c9cc3238> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.DelayQueue.poll(DelayQueue.java:259) at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:323) at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$$Lambda$164/1053864765.run(Unknown Source) at java.lang.Thread.run(Thread.java:750) "AsyncConn Chore Service.Chore.1" #534770 daemon prio=5 os_prio=0 tid=0x00007fc08d5a9800 nid=0xf9b waiting on condition [0x00007fc0406c5000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007fc5c9cc15c8> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1093) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:750) "ReadOnlyZKClient-cod-1e3xsc06g8mjm-master1.exait3p.z1did1.b0.cloudera.site:2181,cod-1e3xsc06g8mjm-master0.exait3p.z1did1.b0.cloudera.site:2181,cod-1e3xsc06g8mjm-leader0.exait3p.z1did1.b0.cloudera.site:2181@0x03312dfe" #534730 daemon prio=5 os_prio=0 tid=0x00007fc075def800 nid=0xf5c waiting on condition [0x00007fc040bca000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007fc5c9cc3898> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.DelayQueue.poll(DelayQueue.java:259) at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:323) at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$$Lambda$164/1053864765.run(Unknown Source) at java.lang.Thread.run(Thread.java:750) "AsyncConn Chore Service.Chore.1" #534729 daemon prio=5 os_prio=0 tid=0x00007fc075def000 nid=0xf5b waiting on condition [0x00007fc040ccb000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007fc5c9cc3a88> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1093) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:750) "ReadOnlyZKClient-cod-1e3xsc06g8mjm-master1.exait3p.z1did1.b0.cloudera.site:2181,cod-1e3xsc06g8mjm-master0.exait3p.z1did1.b0.cloudera.site:2181,cod-1e3xsc06g8mjm-leader0.exait3p.z1did1.b0.cloudera.site:2181@0x37d937a7" #534721 daemon prio=5 os_prio=0 tid=0x00007fc0a6716800 nid=0xf51 waiting on condition [0x00007fc0414d3000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007fc5c9cc3cc8> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.DelayQueue.poll(DelayQueue.java:259) at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:323) at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$$Lambda$164/1053864765.run(Unknown Source) at java.lang.Thread.run(Thread.java:750) "AsyncConn Chore Service.Chore.1" #534720 daemon prio=5 os_prio=0 tid=0x00007fc0b42e5800 nid=0xf50 waiting on condition [0x00007fc0415d4000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007fc5c9cbf040> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1093) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:750) "ReadOnlyZKClient-cod-1e3xsc06g8mjm-master1.exait3p.z1did1.b0.cloudera.site:2181,cod-1e3xsc06g8mjm-master0.exait3p.z1did1.b0.cloudera.site:2181,cod-1e3xsc06g8mjm-leader0.exait3p.z1did1.b0.cloudera.site:2181@0x46034099" #534700 daemon prio=5 os_prio=0 tid=0x00007fc0a61da800 nid=0xf34 waiting on condition [0x00007fc0423e2000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007fc5c9cc3ed8> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.DelayQueue.poll(DelayQueue.java:259) at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:323) at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$$Lambda$164/1053864765.run(Unknown Source) at java.lang.Thread.run(Thread.java:750) "AsyncConn Chore Service.Chore.1" #534699 daemon prio=5 os_prio=0 tid=0x00007fc06c27f000 nid=0xf33 waiting on condition [0x00007fc0424e3000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007fc5c9cbf280> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1093) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:750) "ReadOnlyZKClient-cod-1e3xsc06g8mjm-master1.exait3p.z1did1.b0.cloudera.site:2181,cod-1e3xsc06g8mjm-master0.exait3p.z1did1.b0.cloudera.site:2181,cod-1e3xsc06g8mjm-leader0.exait3p.z1did1.b0.cloudera.site:2181@0x5f69e968" #534695 daemon prio=5 os_prio=0 tid=0x00007fc02cbef000 nid=0xf2f waiting on condition [0x00007fc0428e7000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007fc5c9cc40e8> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.DelayQueue.poll(DelayQueue.java:259) at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:323) at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$$Lambda$164/1053864765.run(Unknown Source) at java.lang.Thread.run(Thread.java:750) "AsyncConn Chore Service.Chore.1" #534694 daemon prio=5 os_prio=0 tid=0x00007fc036098800 nid=0xf2e waiting on condition [0x00007fc0433f2000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007fc5c9cc42d8> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1093) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:750) "ReadOnlyZKClient-cod-1e3xsc06g8mjm-master1.exait3p.z1did1.b0.cloudera.site:2181,cod-1e3xsc06g8mjm-master0.exait3p.z1did1.b0.cloudera.site:2181,cod-1e3xsc06g8mjm-leader0.exait3p.z1did1.b0.cloudera.site:2181@0x3fa661ce" #534675 daemon prio=5 os_prio=0 tid=0x00007fc09e8be000 nid=0xf17 waiting on condition [0x00007fc0439f8000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007fc5c9cc1808> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.DelayQueue.poll(DelayQueue.java:259) at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:323) at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$$Lambda$164/1053864765.run(Unknown Source) at java.lang.Thread.run(Thread.java:750) "AsyncConn Chore Service.Chore.1" #534674 daemon prio=5 os_prio=0 tid=0x00007fc090643800 nid=0xf16 waiting on condition [0x00007fc043af9000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007fc5c9cc4518> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1093) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:750) "ReadOnlyZKClient-cod-1e3xsc06g8mjm-master1.exait3p.z1did1.b0.cloudera.site:2181,cod-1e3xsc06g8mjm-master0.exait3p.z1did1.b0.cloudera.site:2181,cod-1e3xsc06g8mjm-leader0.exait3p.z1did1.b0.cloudera.site:2181@0x31b56449" #534648 daemon prio=5 os_prio=0 tid=0x00007fc02e8a5800 nid=0xeda waiting on condition [0x00007fc043efd000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007fc50d0c7d78> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.DelayQueue.poll(DelayQueue.java:259) at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:323) at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$$Lambda$164/1053864765.run(Unknown Source) at java.lang.Thread.run(Thread.java:750) "AsyncConn Chore Service.Chore.1" #534647 daemon prio=5 os_prio=0 tid=0x00007fc077678800 nid=0xed9 waiting on condition [0x00007fc043ffe000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007fc50d0c7f68> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1093) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:750) "ReadOnlyZKClient-cod-1e3xsc06g8mjm-master1.exait3p.z1did1.b0.cloudera.site:2181,cod-1e3xsc06g8mjm-master0.exait3p.z1did1.b0.cloudera.site:2181,cod-1e3xsc06g8mjm-leader0.exait3p.z1did1.b0.cloudera.site:2181@0x31002449" #534638 daemon prio=5 os_prio=0 tid=0x00007fc0a5ef4800 nid=0xecd waiting on condition [0x00007fc0486c5000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007fc50d0c81a8> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.DelayQueue.poll(DelayQueue.java:259) at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:323) at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$$Lambda$164/1053864765.run(Unknown Source) at java.lang.Thread.run(Thread.java:750) "AsyncConn Chore Service.Chore.1" #534637 daemon prio=5 os_prio=0 tid=0x00007fc0a5ef4000 nid=0xecc waiting on condition [0x00007fc0487c6000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007fc50d0c8398> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1093) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:750) "ReadOnlyZKClient-cod-1e3xsc06g8mjm-master1.exait3p.z1did1.b0.cloudera.site:2181,cod-1e3xsc06g8mjm-master0.exait3p.z1did1.b0.cloudera.site:2181,cod-1e3xsc06g8mjm-leader0.exait3p.z1did1.b0.cloudera.site:2181@0x7b05a6a6" #534621 daemon prio=5 os_prio=0 tid=0x00007fc0a5682000 nid=0xeb9 waiting on condition [0x00007fc0495d4000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007fc50d0c85d8> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.DelayQueue.poll(DelayQueue.java:259) at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:323) at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$$Lambda$164/1053864765.run(Unknown Source) at java.lang.Thread.run(Thread.java:750) "AsyncConn Chore Service.Chore.1" #534620 daemon prio=5 os_prio=0 tid=0x00007fc03cb9a000 nid=0xeb8 waiting on condition [0x00007fc0494d3000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007fc50d0c87c8> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1093) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:750) "ReadOnlyZKClient-cod-1e3xsc06g8mjm-master1.exait3p.z1did1.b0.cloudera.site:2181,cod-1e3xsc06g8mjm-master0.exait3p.z1did1.b0.cloudera.site:2181,cod-1e3xsc06g8mjm-leader0.exait3p.z1did1.b0.cloudera.site:2181@0x6d299964" #534607 daemon prio=5 os_prio=0 tid=0x00007fc0a5cd9800 nid=0xe94 waiting on condition [0x00007fc049bda000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007fc50d0c6408> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.DelayQueue.poll(DelayQueue.java:259) at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:323) at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$$Lambda$164/1053864765.run(Unknown Source) at java.lang.Thread.run(Thread.java:750) "AsyncConn Chore Service.Chore.1" #534606 daemon prio=5 os_prio=0 tid=0x00007fc11ded9800 nid=0xe93 waiting on condition [0x00007fc049cdb000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007fc50d0c8a08> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1093) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:750) "ReadOnlyZKClient-cod-1e3xsc06g8mjm-master1.exait3p.z1did1.b0.cloudera.site:2181,cod-1e3xsc06g8mjm-master0.exait3p.z1did1.b0.cloudera.site:2181,cod-1e3xsc06g8mjm-leader0.exait3p.z1did1.b0.cloudera.site:2181@0x1e3111ca" #534579 daemon prio=5 os_prio=0 tid=0x00007fc02ca05000 nid=0xe62 waiting on condition [0x00007fc04baf9000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007fc50d0c65f8> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.DelayQueue.poll(DelayQueue.java:259) at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:323) at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$$Lambda$164/1053864765.run(Unknown Source) at java.lang.Thread.run(Thread.java:750) "AsyncConn Chore Service.Chore.1" #534578 daemon prio=5 os_prio=0 tid=0x00007fc02ca04800 nid=0xe61 waiting on condition [0x00007fc04b3f2000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007fc50d0c8c48> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1093) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:750) "ReadOnlyZKClient-cod-1e3xsc06g8mjm-master1.exait3p.z1did1.b0.cloudera.site:2181,cod-1e3xsc06g8mjm-master0.exait3p.z1did1.b0.cloudera.site:2181,cod-1e3xsc06g8mjm-leader0.exait3p.z1did1.b0.cloudera.site:2181@0x3d0f9f99" #534572 daemon prio=5 os_prio=0 tid=0x00007fc046e33800 nid=0xe5a waiting on condition [0x00007fc04b0ef000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007fc50d0c6808> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.DelayQueue.poll(DelayQueue.java:259) at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:323) at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$$Lambda$164/1053864765.run(Unknown Source) at java.lang.Thread.run(Thread.java:750) "AsyncConn Chore Service.Chore.1" #534571 daemon prio=5 os_prio=0 tid=0x00007fc046e33000 nid=0xe59 waiting on condition [0x00007fc04b1f0000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007fc50d0c6a18> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1093) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:750) "ReadOnlyZKClient-cod-1e3xsc06g8mjm-master1.exait3p.z1did1.b0.cloudera.site:2181,cod-1e3xsc06g8mjm-master0.exait3p.z1did1.b0.cloudera.site:2181,cod-1e3xsc06g8mjm-leader0.exait3p.z1did1.b0.cloudera.site:2181@0x48039362" #534549 daemon prio=5 os_prio=0 tid=0x00007fc0b4c17800 nid=0xe3f waiting on condition [0x00007fc0506c5000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007fc50d0c8ea8> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.DelayQueue.poll(DelayQueue.java:259) at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:323) at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$$Lambda$164/1053864765.run(Unknown Source) at java.lang.Thread.run(Thread.java:750) "AsyncConn Chore Service.Chore.1" #534548 daemon prio=5 os_prio=0 tid=0x00007fc0b4c16800 nid=0xe3e waiting on condition [0x00007fc0507c6000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007fc50d0c6c78> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1093) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:750) "ReadOnlyZKClient-cod-1e3xsc06g8mjm-master1.exait3p.z1did1.b0.cloudera.site:2181,cod-1e3xsc06g8mjm-master0.exait3p.z1did1.b0.cloudera.site:2181,cod-1e3xsc06g8mjm-leader0.exait3p.z1did1.b0.cloudera.site:2181@0x669d7663" #534544 daemon prio=5 os_prio=0 tid=0x00007fc05567d000 nid=0xe3a waiting on condition [0x00007fc050bca000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007fc50d0c90b8> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.DelayQueue.poll(DelayQueue.java:259) at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:323) at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$$Lambda$164/1053864765.run(Unknown Source) at java.lang.Thread.run(Thread.java:750) "AsyncConn Chore Service.Chore.1" #534543 daemon prio=5 os_prio=0 tid=0x00007fc05567c000 nid=0xe38 waiting on condition [0x00007fc050ccb000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007fc50d0c6ed8> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1093) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:750) "ReadOnlyZKClient-cod-1e3xsc06g8mjm-master1.exait3p.z1did1.b0.cloudera.site:2181,cod-1e3xsc06g8mjm-master0.exait3p.z1did1.b0.cloudera.site:2181,cod-1e3xsc06g8mjm-leader0.exait3p.z1did1.b0.cloudera.site:2181@0x74a55af8" #534530 daemon prio=5 os_prio=0 tid=0x00007fc0aee8d800 nid=0xe2b waiting on condition [0x00007fc053bfa000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007fc50d0c94d8> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.DelayQueue.poll(DelayQueue.java:259) at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:323) at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$$Lambda$164/1053864765.run(Unknown Source) at java.lang.Thread.run(Thread.java:750) "AsyncConn Chore Service.Chore.1" #534529 daemon prio=5 os_prio=0 tid=0x00007fc04635d800 nid=0xe2a waiting on condition [0x00007fc051cdb000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007fc50d0c7398> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1093) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:750) "ReadOnlyZKClient-cod-1e3xsc06g8mjm-master1.exait3p.z1did1.b0.cloudera.site:2181,cod-1e3xsc06g8mjm-master0.exait3p.z1did1.b0.cloudera.site:2181,cod-1e3xsc06g8mjm-leader0.exait3p.z1did1.b0.cloudera.site:2181@0x1d9b6f4f" #534502 daemon prio=5 os_prio=0 tid=0x00007fc06e41a000 nid=0xe0b waiting on condition [0x00007fc0529e8000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007fc50d0c96e8> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.DelayQueue.poll(DelayQueue.java:259) at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:323) at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$$Lambda$164/1053864765.run(Unknown Source) at java.lang.Thread.run(Thread.java:750) "ReadOnlyZKClient-cod-1e3xsc06g8mjm-master1.exait3p.z1did1.b0.cloudera.site:2181,cod-1e3xsc06g8mjm-master0.exait3p.z1did1.b0.cloudera.site:2181,cod-1e3xsc06g8mjm-leader0.exait3p.z1did1.b0.cloudera.site:2181@0x5b7ed638" #534501 daemon prio=5 os_prio=0 tid=0x00007fc06a0c1000 nid=0xe0a waiting on condition [0x00007fc052ae9000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007fc50d0c75f8> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.DelayQueue.poll(DelayQueue.java:259) at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:323) at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$$Lambda$164/1053864765.run(Unknown Source) at java.lang.Thread.run(Thread.java:750) "AsyncConn Chore Service.Chore.1" #534500 daemon prio=5 os_prio=0 tid=0x00007fc99bff3800 nid=0xe09 waiting on condition [0x00007fc052bea000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007fc50d0c4f58> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1093) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:750) "AsyncConn Chore Service.Chore.1" #534499 daemon prio=5 os_prio=0 tid=0x00007fc06a0c0800 nid=0xe08 waiting on condition [0x00007fc052ceb000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007fc50d0c98f8> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1093) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:750) "ReadOnlyZKClient-cod-1e3xsc06g8mjm-master1.exait3p.z1did1.b0.cloudera.site:2181,cod-1e3xsc06g8mjm-master0.exait3p.z1did1.b0.cloudera.site:2181,cod-1e3xsc06g8mjm-leader0.exait3p.z1did1.b0.cloudera.site:2181@0x70560fd1" #534493 daemon prio=5 os_prio=0 tid=0x00007fc02ddcd800 nid=0xe02 waiting on condition [0x00007fc0532f1000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007fc50d0c7808> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.DelayQueue.poll(DelayQueue.java:259) at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:323) at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$$Lambda$164/1053864765.run(Unknown Source) at java.lang.Thread.run(Thread.java:750) "AsyncConn Chore Service.Chore.1" #534491 daemon prio=5 os_prio=0 tid=0x00007fc047d76000 nid=0xe00 waiting on condition [0x00007fc0534f3000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007fc50d0c5198> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1093) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:750) "ReadOnlyZKClient-cod-1e3xsc06g8mjm-master1.exait3p.z1did1.b0.cloudera.site:2181,cod-1e3xsc06g8mjm-master0.exait3p.z1did1.b0.cloudera.site:2181,cod-1e3xsc06g8mjm-leader0.exait3p.z1did1.b0.cloudera.site:2181@0x37088df0" #534482 daemon prio=5 os_prio=0 tid=0x00007fc0a6f9c000 nid=0xde0 waiting on condition [0x00007fc0538f7000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007fc668715fc8> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.DelayQueue.poll(DelayQueue.java:259) at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:323) at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$$Lambda$164/1053864765.run(Unknown Source) at java.lang.Thread.run(Thread.java:750) "AsyncConn Chore Service.Chore.1" #534481 daemon prio=5 os_prio=0 tid=0x00007fc0a6f9b000 nid=0xddf waiting on condition [0x00007fc0539f8000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007fc668713778> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1093) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:750) "ReadOnlyZKClient-cod-1e3xsc06g8mjm-master1.exait3p.z1did1.b0.cloudera.site:2181,cod-1e3xsc06g8mjm-master0.exait3p.z1did1.b0.cloudera.site:2181,cod-1e3xsc06g8mjm-leader0.exait3p.z1did1.b0.cloudera.site:2181@0x17839ebf" #534463 daemon prio=5 os_prio=0 tid=0x00007fc06b23b800 nid=0xdcb waiting on condition [0x00007fc0587c6000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007fc668711778> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.DelayQueue.poll(DelayQueue.java:259) at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:323) at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$$Lambda$164/1053864765.run(Unknown Source) at java.lang.Thread.run(Thread.java:750) "AsyncConn Chore Service.Chore.1" #534462 daemon prio=5 os_prio=0 tid=0x00007fc06b23b000 nid=0xdca waiting on condition [0x00007fc0588c7000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007fc6687139b8> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1093) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:750) "ReadOnlyZKClient-cod-1e3xsc06g8mjm-master1.exait3p.z1did1.b0.cloudera.site:2181,cod-1e3xsc06g8mjm-master0.exait3p.z1did1.b0.cloudera.site:2181,cod-1e3xsc06g8mjm-leader0.exait3p.z1did1.b0.cloudera.site:2181@0x3e0a8537" #534456 daemon prio=5 os_prio=0 tid=0x00007fc0a63bc000 nid=0xdc0 waiting on condition [0x00007fc058ecd000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007fc668711968> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.DelayQueue.poll(DelayQueue.java:259) at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:323) at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$$Lambda$164/1053864765.run(Unknown Source) at java.lang.Thread.run(Thread.java:750) "AsyncConn Chore Service.Chore.1" #534455 daemon prio=5 os_prio=0 tid=0x00007fc06e4af000 nid=0xdbf waiting on condition [0x00007fc0598d7000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007fc668713bf8> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1093) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:750) "ReadOnlyZKClient-cod-1e3xsc06g8mjm-master1.exait3p.z1did1.b0.cloudera.site:2181,cod-1e3xsc06g8mjm-master0.exait3p.z1did1.b0.cloudera.site:2181,cod-1e3xsc06g8mjm-leader0.exait3p.z1did1.b0.cloudera.site:2181@0x62e4415a" #534447 daemon prio=5 os_prio=0 tid=0x00007fc0b5953000 nid=0xdb4 waiting on condition [0x00007fc0595d4000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007fc668711b58> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.DelayQueue.poll(DelayQueue.java:259) at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:323) at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$$Lambda$164/1053864765.run(Unknown Source) at java.lang.Thread.run(Thread.java:750) "AsyncConn Chore Service.Chore.1" #534446 daemon prio=5 os_prio=0 tid=0x00007fc06c06c800 nid=0xdb3 waiting on condition [0x00007fc0596d5000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007fc668713e38> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1093) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:750) "ReadOnlyZKClient-cod-1e3xsc06g8mjm-master1.exait3p.z1did1.b0.cloudera.site:2181,cod-1e3xsc06g8mjm-master0.exait3p.z1did1.b0.cloudera.site:2181,cod-1e3xsc06g8mjm-leader0.exait3p.z1did1.b0.cloudera.site:2181@0x4b0284f9" #534438 daemon prio=5 os_prio=0 tid=0x00007fc068c3b800 nid=0xda8 waiting on condition [0x00007fc059cdb000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007fc668711d48> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.DelayQueue.poll(DelayQueue.java:259) at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:323) at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$$Lambda$164/1053864765.run(Unknown Source) at java.lang.Thread.run(Thread.java:750) "AsyncConn Chore Service.Chore.1" #534437 daemon prio=5 os_prio=0 tid=0x00007fc03584b000 nid=0xda7 waiting on condition [0x00007fc059ddc000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007fc668711f38> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1093) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:750) "ReadOnlyZKClient-cod-1e3xsc06g8mjm-master1.exait3p.z1did1.b0.cloudera.site:2181,cod-1e3xsc06g8mjm-master0.exait3p.z1did1.b0.cloudera.site:2181,cod-1e3xsc06g8mjm-leader0.exait3p.z1did1.b0.cloudera.site:2181@0x2744c648" #534428 daemon prio=5 os_prio=0 tid=0x00007fc035dab800 nid=0xd9d waiting on condition [0x00007fc05a5e4000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007fc668712178> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.DelayQueue.poll(DelayQueue.java:259) at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:323) at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$$Lambda$164/1053864765.run(Unknown Source) at java.lang.Thread.run(Thread.java:750) "AsyncConn Chore Service.Chore.1" #534427 daemon prio=5 os_prio=0 tid=0x00007fc035dab000 nid=0xd9c waiting on condition [0x00007fc05aae9000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007fc66871f7f8> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1093) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:750) "ReadOnlyZKClient-cod-1e3xsc06g8mjm-master1.exait3p.z1did1.b0.cloudera.site:2181,cod-1e3xsc06g8mjm-master0.exait3p.z1did1.b0.cloudera.site:2181,cod-1e3xsc06g8mjm-leader0.exait3p.z1did1.b0.cloudera.site:2181@0x6830eb43" #534396 daemon prio=5 os_prio=0 tid=0x00007fc0872e4800 nid=0xd77 waiting on condition [0x00007fc05bcfb000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007fc668712368> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.DelayQueue.poll(DelayQueue.java:259) at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:323) at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$$Lambda$164/1053864765.run(Unknown Source) at java.lang.Thread.run(Thread.java:750) "AsyncConn Chore Service.Chore.1" #534395 daemon prio=5 os_prio=0 tid=0x00007fc986d44800 nid=0xd76 waiting on condition [0x00007fc0604c3000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007fc66871fa38> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1093) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:750) "ReadOnlyZKClient-cod-1e3xsc06g8mjm-master1.exait3p.z1did1.b0.cloudera.site:2181,cod-1e3xsc06g8mjm-master0.exait3p.z1did1.b0.cloudera.site:2181,cod-1e3xsc06g8mjm-leader0.exait3p.z1did1.b0.cloudera.site:2181@0x2ffc5030" #534389 daemon prio=5 os_prio=0 tid=0x00007fc0a56ff800 nid=0xd70 waiting on condition [0x00007fc0602c1000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007fc668712558> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.DelayQueue.poll(DelayQueue.java:259) at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:323) at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$$Lambda$164/1053864765.run(Unknown Source) at java.lang.Thread.run(Thread.java:750) "AsyncConn Chore Service.Chore.1" #534388 daemon prio=5 os_prio=0 tid=0x00007fc0a56ff000 nid=0xd6f waiting on condition [0x00007fc0603c2000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007fc66871e018> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1093) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:750) "ReadOnlyZKClient-cod-1e3xsc06g8mjm-master1.exait3p.z1did1.b0.cloudera.site:2181,cod-1e3xsc06g8mjm-master0.exait3p.z1did1.b0.cloudera.site:2181,cod-1e3xsc06g8mjm-leader0.exait3p.z1did1.b0.cloudera.site:2181@0x06effd80" #534381 daemon prio=5 os_prio=0 tid=0x00007fc06c195000 nid=0xd4c waiting on condition [0x00007fc060ac9000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007fc668712748> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.DelayQueue.poll(DelayQueue.java:259) at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:323) at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$$Lambda$164/1053864765.run(Unknown Source) at java.lang.Thread.run(Thread.java:750) "AsyncConn Chore Service.Chore.1" #534380 daemon prio=5 os_prio=0 tid=0x00007fc06c194000 nid=0xd4b waiting on condition [0x00007fc060bca000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007fc66871fc78> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1093) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:750) "ReadOnlyZKClient-cod-1e3xsc06g8mjm-master1.exait3p.z1did1.b0.cloudera.site:2181,cod-1e3xsc06g8mjm-master0.exait3p.z1did1.b0.cloudera.site:2181,cod-1e3xsc06g8mjm-leader0.exait3p.z1did1.b0.cloudera.site:2181@0x478cd999" #534369 daemon prio=5 os_prio=0 tid=0x00007fc0a7588800 nid=0xd40 waiting on condition [0x00007fc0615d4000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007fc668712938> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.DelayQueue.poll(DelayQueue.java:259) at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:323) at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$$Lambda$164/1053864765.run(Unknown Source) at java.lang.Thread.run(Thread.java:750) "AsyncConn Chore Service.Chore.1" #534368 daemon prio=5 os_prio=0 tid=0x00007fc11fcaa000 nid=0xd3f waiting on condition [0x00007fc061ad9000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007fc66871feb8> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1093) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:750) "ReadOnlyZKClient-cod-1e3xsc06g8mjm-master1.exait3p.z1did1.b0.cloudera.site:2181,cod-1e3xsc06g8mjm-master0.exait3p.z1did1.b0.cloudera.site:2181,cod-1e3xsc06g8mjm-leader0.exait3p.z1did1.b0.cloudera.site:2181@0x4b07b323" #534333 daemon prio=5 os_prio=0 tid=0x00007fc05c665800 nid=0xd07 waiting on condition [0x00007fc062ceb000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007fc66871e258> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.DelayQueue.poll(DelayQueue.java:259) at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:323) at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$$Lambda$164/1053864765.run(Unknown Source) at java.lang.Thread.run(Thread.java:750) "AsyncConn Chore Service.Chore.1" #534332 daemon prio=5 os_prio=0 tid=0x00007fc05c665000 nid=0xd06 waiting on condition [0x00007fc062dec000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007fc6687200f8> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1093) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:750) "ReadOnlyZKClient-cod-1e3xsc06g8mjm-master1.exait3p.z1did1.b0.cloudera.site:2181,cod-1e3xsc06g8mjm-master0.exait3p.z1did1.b0.cloudera.site:2181,cod-1e3xsc06g8mjm-leader0.exait3p.z1did1.b0.cloudera.site:2181@0x6e9fe289" #534321 daemon prio=5 os_prio=0 tid=0x00007fc0ae246000 nid=0xce3 waiting on condition [0x00007fc0636f5000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007fc668712b28> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.DelayQueue.poll(DelayQueue.java:259) at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:323) at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$$Lambda$164/1053864765.run(Unknown Source) at java.lang.Thread.run(Thread.java:750) "AsyncConn Chore Service.Chore.1" #534320 daemon prio=5 os_prio=0 tid=0x00007fc06a025000 nid=0xce2 waiting on condition [0x00007fc0637f6000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007fc66871e448> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1093) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:750) "ReadOnlyZKClient-cod-1e3xsc06g8mjm-master1.exait3p.z1did1.b0.cloudera.site:2181,cod-1e3xsc06g8mjm-master0.exait3p.z1did1.b0.cloudera.site:2181,cod-1e3xsc06g8mjm-leader0.exait3p.z1did1.b0.cloudera.site:2181@0x757e19db" #534315 daemon prio=5 os_prio=0 tid=0x00007fc0a621c800 nid=0xcd3 waiting on condition [0x00007fc063cfb000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007fc668720338> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.DelayQueue.poll(DelayQueue.java:259) at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:323) at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$$Lambda$164/1053864765.run(Unknown Source) at java.lang.Thread.run(Thread.java:750) "AsyncConn Chore Service.Chore.1" #534314 daemon prio=5 os_prio=0 tid=0x00007fc0a621c000 nid=0xcd2 waiting on condition [0x00007fc0709c8000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007fc668712d18> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1093) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:750) "ReadOnlyZKClient-cod-1e3xsc06g8mjm-master1.exait3p.z1did1.b0.cloudera.site:2181,cod-1e3xsc06g8mjm-master0.exait3p.z1did1.b0.cloudera.site:2181,cod-1e3xsc06g8mjm-leader0.exait3p.z1did1.b0.cloudera.site:2181@0x3b6dd06e" #534292 daemon prio=5 os_prio=0 tid=0x00007fc0b4a3d000 nid=0xcb4 waiting on condition [0x00007fc0711d0000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007fc668720528> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.DelayQueue.poll(DelayQueue.java:259) at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:323) at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$$Lambda$164/1053864765.run(Unknown Source) at java.lang.Thread.run(Thread.java:750) "AsyncConn Chore Service.Chore.1" #534291 daemon prio=5 os_prio=0 tid=0x00007fc0b491b800 nid=0xcb3 waiting on condition [0x00007fc0716d5000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007fc66871e688> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1093) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:750) "ReadOnlyZKClient-cod-1e3xsc06g8mjm-master1.exait3p.z1did1.b0.cloudera.site:2181,cod-1e3xsc06g8mjm-master0.exait3p.z1did1.b0.cloudera.site:2181,cod-1e3xsc06g8mjm-leader0.exait3p.z1did1.b0.cloudera.site:2181@0x2e4bdc4b" #534265 daemon prio=5 os_prio=0 tid=0x00007fc0a6f1b000 nid=0xc94 waiting on condition [0x00007fc0722e1000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007fc668720718> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.DelayQueue.poll(DelayQueue.java:259) at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:323) at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$$Lambda$164/1053864765.run(Unknown Source) at java.lang.Thread.run(Thread.java:750) "AsyncConn Chore Service.Chore.1" #534264 daemon prio=5 os_prio=0 tid=0x00007fc0b70ab800 nid=0xc93 waiting on condition [0x00007fc0723e2000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007fc668720908> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1093) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:750) "ReadOnlyZKClient-cod-1e3xsc06g8mjm-master1.exait3p.z1did1.b0.cloudera.site:2181,cod-1e3xsc06g8mjm-master0.exait3p.z1did1.b0.cloudera.site:2181,cod-1e3xsc06g8mjm-leader0.exait3p.z1did1.b0.cloudera.site:2181@0x4ee50b9f" #534231 daemon prio=5 os_prio=0 tid=0x00007fc056752800 nid=0xc57 waiting on condition [0x00007fc0788c7000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007fc66871e8c8> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.DelayQueue.poll(DelayQueue.java:259) at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:323) at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$$Lambda$164/1053864765.run(Unknown Source) at java.lang.Thread.run(Thread.java:750) "AsyncConn Chore Service.Chore.1" #534230 daemon prio=5 os_prio=0 tid=0x00007fc056752000 nid=0xc56 waiting on condition [0x00007fc0789c8000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007fc668714078> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1093) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:750) "ReadOnlyZKClient-cod-1e3xsc06g8mjm-master1.exait3p.z1did1.b0.cloudera.site:2181,cod-1e3xsc06g8mjm-master0.exait3p.z1did1.b0.cloudera.site:2181,cod-1e3xsc06g8mjm-leader0.exait3p.z1did1.b0.cloudera.site:2181@0x3c20f53d" #534226 daemon prio=5 os_prio=0 tid=0x00007fc0a7788800 nid=0xc52 waiting on condition [0x00007fc0797d6000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007fc668720b48> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.DelayQueue.poll(DelayQueue.java:259) at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:323) at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$$Lambda$164/1053864765.run(Unknown Source) at java.lang.Thread.run(Thread.java:750) "AsyncConn Chore Service.Chore.1" #534225 daemon prio=5 os_prio=0 tid=0x00007fc0a6455800 nid=0xc51 waiting on condition [0x00007fc0798d7000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007fc66871eab8> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1093) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:750) "ReadOnlyZKClient-cod-1e3xsc06g8mjm-master1.exait3p.z1did1.b0.cloudera.site:2181,cod-1e3xsc06g8mjm-master0.exait3p.z1did1.b0.cloudera.site:2181,cod-1e3xsc06g8mjm-leader0.exait3p.z1did1.b0.cloudera.site:2181@0x648169ee" #534220 daemon prio=5 os_prio=0 tid=0x00007fc0a56ea800 nid=0xc4c waiting on condition [0x00007fc079fde000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007fc6687142b8> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.DelayQueue.poll(DelayQueue.java:259) at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:323) at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$$Lambda$164/1053864765.run(Unknown Source) at java.lang.Thread.run(Thread.java:750) "AsyncConn Chore Service.Chore.1" #534219 daemon prio=5 os_prio=0 tid=0x00007fc0a56ea000 nid=0xc4b waiting on condition [0x00007fc07a7e6000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007fc668720d38> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1093) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:750) "ReadOnlyZKClient-cod-1e3xsc06g8mjm-master1.exait3p.z1did1.b0.cloudera.site:2181,cod-1e3xsc06g8mjm-master0.exait3p.z1did1.b0.cloudera.site:2181,cod-1e3xsc06g8mjm-leader0.exait3p.z1did1.b0.cloudera.site:2181@0x01fe9575" #534216 daemon prio=5 os_prio=0 tid=0x00007fc11e3a8800 nid=0xc48 waiting on condition [0x00007fc07aae9000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007fc668720f78> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.DelayQueue.poll(DelayQueue.java:259) at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:323) at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$$Lambda$164/1053864765.run(Unknown Source) at java.lang.Thread.run(Thread.java:750) "AsyncConn Chore Service.Chore.1" #534215 daemon prio=5 os_prio=0 tid=0x00007fc11e3a8000 nid=0xc47 waiting on condition [0x00007fc07aceb000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007fc66871ecf8> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1093) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:750) "ReadOnlyZKClient-cod-1e3xsc06g8mjm-master1.exait3p.z1did1.b0.cloudera.site:2181,cod-1e3xsc06g8mjm-master0.exait3p.z1did1.b0.cloudera.site:2181,cod-1e3xsc06g8mjm-leader0.exait3p.z1did1.b0.cloudera.site:2181@0x50161db7" #534211 daemon prio=5 os_prio=0 tid=0x00007fc02ea1a800 nid=0xc41 waiting on condition [0x00007fc07b5f4000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007fc668721168> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.DelayQueue.poll(DelayQueue.java:259) at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:323) at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$$Lambda$164/1053864765.run(Unknown Source) at java.lang.Thread.run(Thread.java:750) "AsyncConn Chore Service.Chore.1" #534210 daemon prio=5 os_prio=0 tid=0x00007fc02e396800 nid=0xc40 waiting on condition [0x00007fc07b6f5000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007fc66871ef38> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1093) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:750) "ReadOnlyZKClient-cod-1e3xsc06g8mjm-master1.exait3p.z1did1.b0.cloudera.site:2181,cod-1e3xsc06g8mjm-master0.exait3p.z1did1.b0.cloudera.site:2181,cod-1e3xsc06g8mjm-leader0.exait3p.z1did1.b0.cloudera.site:2181@0x781e8e2e" #534205 daemon prio=5 os_prio=0 tid=0x00007fc0367fb000 nid=0xc3b waiting on condition [0x00007fc07bcfb000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007fc668721358> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.DelayQueue.poll(DelayQueue.java:259) at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:323) at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$$Lambda$164/1053864765.run(Unknown Source) at java.lang.Thread.run(Thread.java:750) "AsyncConn Chore Service.Chore.1" #534204 daemon prio=5 os_prio=0 tid=0x00007fc056544800 nid=0xc3a waiting on condition [0x00007fc07befd000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007fc668721548> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1093) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:750) "ReadOnlyZKClient-cod-1e3xsc06g8mjm-master1.exait3p.z1did1.b0.cloudera.site:2181,cod-1e3xsc06g8mjm-master0.exait3p.z1did1.b0.cloudera.site:2181,cod-1e3xsc06g8mjm-leader0.exait3p.z1did1.b0.cloudera.site:2181@0x15b2287b" #534198 daemon prio=5 os_prio=0 tid=0x00007fc0a680d800 nid=0xc34 waiting on condition [0x00007fc117e10000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007fc66871f178> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.DelayQueue.poll(DelayQueue.java:259) at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:323) at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$$Lambda$164/1053864765.run(Unknown Source) at java.lang.Thread.run(Thread.java:750) "AsyncConn Chore Service.Chore.1" #534197 daemon prio=5 os_prio=0 tid=0x00007fc11dff0000 nid=0xc33 waiting on condition [0x00007fc0cf9fa000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007fc6687217a8> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1093) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:750) "ReadOnlyZKClient-cod-1e3xsc06g8mjm-master1.exait3p.z1did1.b0.cloudera.site:2181,cod-1e3xsc06g8mjm-master0.exait3p.z1did1.b0.cloudera.site:2181,cod-1e3xsc06g8mjm-leader0.exait3p.z1did1.b0.cloudera.site:2181@0x61cd2fb0" #534193 daemon prio=5 os_prio=0 tid=0x0000558b2e007000 nid=0xc2f waiting on condition [0x00007fc0994d3000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007fc66871f368> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.DelayQueue.poll(DelayQueue.java:259) at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:323) at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$$Lambda$164/1053864765.run(Unknown Source) at java.lang.Thread.run(Thread.java:750) "AsyncConn Chore Service.Chore.1" #534192 daemon prio=5 os_prio=0 tid=0x0000558b2e006000 nid=0xc2e waiting on condition [0x00007fc0995d4000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007fc6687219e8> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1093) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:750) "ReadOnlyZKClient-cod-1e3xsc06g8mjm-master1.exait3p.z1did1.b0.cloudera.site:2181,cod-1e3xsc06g8mjm-master0.exait3p.z1did1.b0.cloudera.site:2181,cod-1e3xsc06g8mjm-leader0.exait3p.z1did1.b0.cloudera.site:2181@0x7b358531" #534189 daemon prio=5 os_prio=0 tid=0x00007fc0b0e67000 nid=0xc2a waiting on condition [0x00007fc099edd000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007fc668721c28> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.DelayQueue.poll(DelayQueue.java:259) at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:323) at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$$Lambda$164/1053864765.run(Unknown Source) at java.lang.Thread.run(Thread.java:750) "ReadOnlyZKClient-cod-1e3xsc06g8mjm-master1.exait3p.z1did1.b0.cloudera.site:2181,cod-1e3xsc06g8mjm-master0.exait3p.z1did1.b0.cloudera.site:2181,cod-1e3xsc06g8mjm-leader0.exait3p.z1did1.b0.cloudera.site:2181@0x07b4ba80" #534186 daemon prio=5 os_prio=0 tid=0x00007fc11dfef000 nid=0xc27 waiting on condition [0x00007fc09a3e2000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007fc668721e18> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.DelayQueue.poll(DelayQueue.java:259) at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:323) at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$$Lambda$164/1053864765.run(Unknown Source) at java.lang.Thread.run(Thread.java:750) "AsyncConn Chore Service.Chore.1" #534185 daemon prio=5 os_prio=0 tid=0x00007fc0b0c17800 nid=0xc25 waiting on condition [0x00007fc09a5e4000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007fc668722008> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1093) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:750) "AsyncConn Chore Service.Chore.1" #534184 daemon prio=5 os_prio=0 tid=0x00007fc11dcfa000 nid=0xc24 waiting on condition [0x00007fc09aae9000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007fc668722248> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1093) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:750) "ReadOnlyZKClient-cod-1e3xsc06g8mjm-master1.exait3p.z1did1.b0.cloudera.site:2181,cod-1e3xsc06g8mjm-master0.exait3p.z1did1.b0.cloudera.site:2181,cod-1e3xsc06g8mjm-leader0.exait3p.z1did1.b0.cloudera.site:2181@0x34e0bd56" #534179 daemon prio=5 os_prio=0 tid=0x00007fc0b3ec6000 nid=0xc20 waiting on condition [0x00007fc078ccb000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007fc6687161b8> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.DelayQueue.poll(DelayQueue.java:259) at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:323) at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$$Lambda$164/1053864765.run(Unknown Source) at java.lang.Thread.run(Thread.java:750) "AsyncConn Chore Service.Chore.1" #534178 daemon prio=5 os_prio=0 tid=0x00007fc07f095800 nid=0xc1f waiting on condition [0x00007fc0782c1000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007fc6687163a8> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1093) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:750) "ReadOnlyZKClient-cod-1e3xsc06g8mjm-master1.exait3p.z1did1.b0.cloudera.site:2181,cod-1e3xsc06g8mjm-master0.exait3p.z1did1.b0.cloudera.site:2181,cod-1e3xsc06g8mjm-leader0.exait3p.z1did1.b0.cloudera.site:2181@0x5ffe7303" #534174 daemon prio=5 os_prio=0 tid=0x00007fc0942dc800 nid=0xc1b waiting on condition [0x00007fc0bbbfa000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007fc668722488> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.DelayQueue.poll(DelayQueue.java:259) at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:323) at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$$Lambda$164/1053864765.run(Unknown Source) at java.lang.Thread.run(Thread.java:750) "AsyncConn Chore Service.Chore.1" #534173 daemon prio=5 os_prio=0 tid=0x00007fc094adc000 nid=0xc1a waiting on condition [0x00007fc073dfc000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007fc6687165e8> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1093) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:750) "ReadOnlyZKClient-cod-1e3xsc06g8mjm-master1.exait3p.z1did1.b0.cloudera.site:2181,cod-1e3xsc06g8mjm-master0.exait3p.z1did1.b0.cloudera.site:2181,cod-1e3xsc06g8mjm-leader0.exait3p.z1did1.b0.cloudera.site:2181@0x45824a78" #534168 daemon prio=5 os_prio=0 tid=0x00007fc0d196a800 nid=0xc14 waiting on condition [0x00007fc07abea000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007fc668716828> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.DelayQueue.poll(DelayQueue.java:259) at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:323) at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$$Lambda$164/1053864765.run(Unknown Source) at java.lang.Thread.run(Thread.java:750) "AsyncConn Chore Service.Chore.1" #534167 daemon prio=5 os_prio=0 tid=0x00007fc0aedb3800 nid=0xc13 waiting on condition [0x00007fc07afee000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007fc668722678> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1093) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:750) "ReadOnlyZKClient-cod-1e3xsc06g8mjm-master1.exait3p.z1did1.b0.cloudera.site:2181,cod-1e3xsc06g8mjm-master0.exait3p.z1did1.b0.cloudera.site:2181,cod-1e3xsc06g8mjm-leader0.exait3p.z1did1.b0.cloudera.site:2181@0x7448f59e" #534162 daemon prio=5 os_prio=0 tid=0x0000558b3056f000 nid=0xc0e waiting on condition [0x00007fc07bffe000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007fc668716a18> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.DelayQueue.poll(DelayQueue.java:259) at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:323) at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$$Lambda$164/1053864765.run(Unknown Source) at java.lang.Thread.run(Thread.java:750) "AsyncConn Chore Service.Chore.1" #534161 daemon prio=5 os_prio=0 tid=0x0000558b30426000 nid=0xc0d waiting on condition [0x00007fc0987c6000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007fc6687228b8> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1093) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:750) "ReadOnlyZKClient-cod-1e3xsc06g8mjm-master1.exait3p.z1did1.b0.cloudera.site:2181,cod-1e3xsc06g8mjm-master0.exait3p.z1did1.b0.cloudera.site:2181,cod-1e3xsc06g8mjm-leader0.exait3p.z1did1.b0.cloudera.site:2181@0x19602090" #534158 daemon prio=5 os_prio=0 tid=0x00007fc0a53fb800 nid=0xc0a waiting on condition [0x00007fc07b3f2000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007fc668717e18> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.DelayQueue.poll(DelayQueue.java:259) at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:323) at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$$Lambda$164/1053864765.run(Unknown Source) at java.lang.Thread.run(Thread.java:750) "AsyncConn Chore Service.Chore.1" #534157 daemon prio=5 os_prio=0 tid=0x00007fc03d638800 nid=0xc09 waiting on condition [0x00007fc07b0ef000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007fc668722af8> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1093) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:750) "ReadOnlyZKClient-cod-1e3xsc06g8mjm-master1.exait3p.z1did1.b0.cloudera.site:2181,cod-1e3xsc06g8mjm-master0.exait3p.z1did1.b0.cloudera.site:2181,cod-1e3xsc06g8mjm-leader0.exait3p.z1did1.b0.cloudera.site:2181@0x7c90fb86" #534152 daemon prio=5 os_prio=0 tid=0x0000558b30995000 nid=0xc04 waiting on condition [0x00007fc0998d7000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007fc668716c08> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.DelayQueue.poll(DelayQueue.java:259) at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:323) at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$$Lambda$164/1053864765.run(Unknown Source) at java.lang.Thread.run(Thread.java:750) "AsyncConn Chore Service.Chore.1" #534151 daemon prio=5 os_prio=0 tid=0x0000558b14149800 nid=0xc03 waiting on condition [0x00007fc0c4642000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007fc668718008> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1093) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:750) "ReadOnlyZKClient-cod-1e3xsc06g8mjm-master1.exait3p.z1did1.b0.cloudera.site:2181,cod-1e3xsc06g8mjm-master0.exait3p.z1did1.b0.cloudera.site:2181,cod-1e3xsc06g8mjm-leader0.exait3p.z1did1.b0.cloudera.site:2181@0x434b5c11" #534145 daemon prio=5 os_prio=0 tid=0x00007fc02d107000 nid=0xbfa waiting on condition [0x00007fc0c76f5000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007fc668716df8> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.DelayQueue.poll(DelayQueue.java:259) at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:323) at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$$Lambda$164/1053864765.run(Unknown Source) at java.lang.Thread.run(Thread.java:750) "AsyncConn Chore Service.Chore.1" #534144 daemon prio=5 os_prio=0 tid=0x00007fc09ce4d000 nid=0xbf9 waiting on condition [0x00007fc0b9ad9000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007fc668722d38> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1093) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:750) "ReadOnlyZKClient-cod-1e3xsc06g8mjm-master1.exait3p.z1did1.b0.cloudera.site:2181,cod-1e3xsc06g8mjm-master0.exait3p.z1did1.b0.cloudera.site:2181,cod-1e3xsc06g8mjm-leader0.exait3p.z1did1.b0.cloudera.site:2181@0x6a2123ac" #533920 daemon prio=5 os_prio=0 tid=0x00007fc02c25d800 nid=0xa99 waiting on condition [0x00007fc098fce000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007fc668716fe8> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.DelayQueue.poll(DelayQueue.java:259) at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:323) at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$$Lambda$164/1053864765.run(Unknown Source) at java.lang.Thread.run(Thread.java:750) "AsyncConn Chore Service.Chore.1" #533919 daemon prio=5 os_prio=0 tid=0x00007fc02e4d0000 nid=0xa98 waiting on condition [0x00007fc0990cf000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007fc668718248> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1093) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:750) "ReadOnlyZKClient-cod-1e3xsc06g8mjm-master1.exait3p.z1did1.b0.cloudera.site:2181,cod-1e3xsc06g8mjm-master0.exait3p.z1did1.b0.cloudera.site:2181,cod-1e3xsc06g8mjm-leader0.exait3p.z1did1.b0.cloudera.site:2181@0x7c7c882b" #533866 daemon prio=5 os_prio=0 tid=0x00007fc0b425c000 nid=0xa48 waiting on condition [0x00007fc09b9f8000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007fc668722f78> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.DelayQueue.poll(DelayQueue.java:259) at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:323) at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$$Lambda$164/1053864765.run(Unknown Source) at java.lang.Thread.run(Thread.java:750) "AsyncConn Chore Service.Chore.1" #533865 daemon prio=5 os_prio=0 tid=0x00007fc0a6249000 nid=0xa47 waiting on condition [0x00007fc09baf9000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007fc6687171d8> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1093) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:750) "ReadOnlyZKClient-cod-1e3xsc06g8mjm-master1.exait3p.z1did1.b0.cloudera.site:2181,cod-1e3xsc06g8mjm-master0.exait3p.z1did1.b0.cloudera.site:2181,cod-1e3xsc06g8mjm-leader0.exait3p.z1did1.b0.cloudera.site:2181@0x509b16ea" #533857 daemon prio=5 os_prio=0 tid=0x00007fc11d141800 nid=0xa3c waiting on condition [0x00007fc09bffe000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007fc668718488> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.DelayQueue.poll(DelayQueue.java:259) at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:323) at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$$Lambda$164/1053864765.run(Unknown Source) at java.lang.Thread.run(Thread.java:750) "AsyncConn Chore Service.Chore.1" #533856 daemon prio=5 os_prio=0 tid=0x00007fc03f55f000 nid=0xa3b waiting on condition [0x00007fc0a01c0000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007fc668723168> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1093) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:750) "ReadOnlyZKClient-cod-1e3xsc06g8mjm-master1.exait3p.z1did1.b0.cloudera.site:2181,cod-1e3xsc06g8mjm-master0.exait3p.z1did1.b0.cloudera.site:2181,cod-1e3xsc06g8mjm-leader0.exait3p.z1did1.b0.cloudera.site:2181@0x0896a85c" #533808 daemon prio=5 os_prio=0 tid=0x00007fc05d366000 nid=0xa05 waiting on condition [0x00007fc0a22e1000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007fc668717418> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.DelayQueue.poll(DelayQueue.java:259) at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:323) at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$$Lambda$164/1053864765.run(Unknown Source) at java.lang.Thread.run(Thread.java:750) "AsyncConn Chore Service.Chore.1" #533807 daemon prio=5 os_prio=0 tid=0x00007fc0a5891800 nid=0xa04 waiting on condition [0x00007fc0a23e2000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007fc66871b778> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1093) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:750) "ReadOnlyZKClient-cod-1e3xsc06g8mjm-master1.exait3p.z1did1.b0.cloudera.site:2181,cod-1e3xsc06g8mjm-master0.exait3p.z1did1.b0.cloudera.site:2181,cod-1e3xsc06g8mjm-leader0.exait3p.z1did1.b0.cloudera.site:2181@0x7b72b55f" #533804 daemon prio=5 os_prio=0 tid=0x0000558b1748e000 nid=0xa01 waiting on condition [0x00007fc0a27e6000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007fc668718678> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.DelayQueue.poll(DelayQueue.java:259) at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:323) at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$$Lambda$164/1053864765.run(Unknown Source) at java.lang.Thread.run(Thread.java:750) "AsyncConn Chore Service.Chore.1" #533803 daemon prio=5 os_prio=0 tid=0x0000558b2e930000 nid=0xa00 waiting on condition [0x00007fc0a28e7000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007fc6687233a8> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1093) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:750) "ReadOnlyZKClient-cod-1e3xsc06g8mjm-master1.exait3p.z1did1.b0.cloudera.site:2181,cod-1e3xsc06g8mjm-master0.exait3p.z1did1.b0.cloudera.site:2181,cod-1e3xsc06g8mjm-leader0.exait3p.z1did1.b0.cloudera.site:2181@0x75ed1574" #533799 daemon prio=5 os_prio=0 tid=0x00007fc11db20000 nid=0x9fc waiting on condition [0x00007fc0babea000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007fc66871b9b8> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.DelayQueue.poll(DelayQueue.java:259) at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:323) at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$$Lambda$164/1053864765.run(Unknown Source) at java.lang.Thread.run(Thread.java:750) "AsyncConn Chore Service.Chore.1" #533798 daemon prio=5 os_prio=0 tid=0x00007fc0b7020000 nid=0x9fb waiting on condition [0x00007fc0badec000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007fc668718868> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1093) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:750) "ReadOnlyZKClient-cod-1e3xsc06g8mjm-master1.exait3p.z1did1.b0.cloudera.site:2181,cod-1e3xsc06g8mjm-master0.exait3p.z1did1.b0.cloudera.site:2181,cod-1e3xsc06g8mjm-leader0.exait3p.z1did1.b0.cloudera.site:2181@0x4e5a17eb" #533785 daemon prio=5 os_prio=0 tid=0x00007fc97e476000 nid=0x9c1 waiting on condition [0x00007fc0e5618000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007fc66871bba8> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.DelayQueue.poll(DelayQueue.java:259) at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:323) at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$$Lambda$164/1053864765.run(Unknown Source) at java.lang.Thread.run(Thread.java:750) "AsyncConn Chore Service.Chore.1" #533784 daemon prio=5 os_prio=0 tid=0x00007fc0b1ab2000 nid=0x9c0 waiting on condition [0x00007fc0e8446000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007fc668718aa8> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1093) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:750) "ReadOnlyZKClient-cod-1e3xsc06g8mjm-master1.exait3p.z1did1.b0.cloudera.site:2181,cod-1e3xsc06g8mjm-master0.exait3p.z1did1.b0.cloudera.site:2181,cod-1e3xsc06g8mjm-leader0.exait3p.z1did1.b0.cloudera.site:2181@0x696d44df" #533781 daemon prio=5 os_prio=0 tid=0x00007fc0aaefe800 nid=0x9bd waiting on condition [0x00007fc0a29e8000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007fc66871bd98> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.DelayQueue.poll(DelayQueue.java:259) at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:323) at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$$Lambda$164/1053864765.run(Unknown Source) at java.lang.Thread.run(Thread.java:750) "AsyncConn Chore Service.Chore.1" #533780 daemon prio=5 os_prio=0 tid=0x00007fc02ebd7000 nid=0x9bc waiting on condition [0x00007fc0a2ae9000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007fc668718ce8> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1093) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:750) "ReadOnlyZKClient-cod-1e3xsc06g8mjm-master1.exait3p.z1did1.b0.cloudera.site:2181,cod-1e3xsc06g8mjm-master0.exait3p.z1did1.b0.cloudera.site:2181,cod-1e3xsc06g8mjm-leader0.exait3p.z1did1.b0.cloudera.site:2181@0x48ae20b7" #533771 daemon prio=5 os_prio=0 tid=0x00007fc0b4909800 nid=0x9b2 waiting on condition [0x00007fc0a2dec000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007fc66871bf88> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.DelayQueue.poll(DelayQueue.java:259) at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:323) at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$$Lambda$164/1053864765.run(Unknown Source) at java.lang.Thread.run(Thread.java:750) "AsyncConn Chore Service.Chore.1" #533770 daemon prio=5 os_prio=0 tid=0x00007fc0b48d1000 nid=0x9b1 waiting on condition [0x00007fc0c3021000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007fc668723778> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1093) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:750) "ReadOnlyZKClient-cod-1e3xsc06g8mjm-master1.exait3p.z1did1.b0.cloudera.site:2181,cod-1e3xsc06g8mjm-master0.exait3p.z1did1.b0.cloudera.site:2181,cod-1e3xsc06g8mjm-leader0.exait3p.z1did1.b0.cloudera.site:2181@0x7a13f91a" #533764 daemon prio=5 os_prio=0 tid=0x00007fc986c7e000 nid=0x9aa waiting on condition [0x00007fc0e25e8000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007fc668718f28> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.DelayQueue.poll(DelayQueue.java:259) at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:323) at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$$Lambda$164/1053864765.run(Unknown Source) at java.lang.Thread.run(Thread.java:750) "AsyncConn Chore Service.Chore.1" #533763 daemon prio=5 os_prio=0 tid=0x00007fc047e85000 nid=0x9a9 waiting on condition [0x00007fc0c1203000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007fc6687239b8> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1093) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:750) "ReadOnlyZKClient-cod-1e3xsc06g8mjm-master1.exait3p.z1did1.b0.cloudera.site:2181,cod-1e3xsc06g8mjm-master0.exait3p.z1did1.b0.cloudera.site:2181,cod-1e3xsc06g8mjm-leader0.exait3p.z1did1.b0.cloudera.site:2181@0x1dfdc6e0" #533760 daemon prio=5 os_prio=0 tid=0x00007fc08b42d800 nid=0x9a6 waiting on condition [0x00007fc0ba8e7000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007fc668719118> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.DelayQueue.poll(DelayQueue.java:259) at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:323) at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$$Lambda$164/1053864765.run(Unknown Source) at java.lang.Thread.run(Thread.java:750) "AsyncConn Chore Service.Chore.1" #533759 daemon prio=5 os_prio=0 tid=0x00007fc08a252000 nid=0x9a5 waiting on condition [0x00007fc0bb3f2000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007fc668719308> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1093) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:750) "ReadOnlyZKClient-cod-1e3xsc06g8mjm-master1.exait3p.z1did1.b0.cloudera.site:2181,cod-1e3xsc06g8mjm-master0.exait3p.z1did1.b0.cloudera.site:2181,cod-1e3xsc06g8mjm-leader0.exait3p.z1did1.b0.cloudera.site:2181@0x087c6e28" #533747 daemon prio=5 os_prio=0 tid=0x00007fc090ad9800 nid=0x976 waiting on condition [0x00007fc0cfffe000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007fc668723bf8> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.DelayQueue.poll(DelayQueue.java:259) at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:323) at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$$Lambda$164/1053864765.run(Unknown Source) at java.lang.Thread.run(Thread.java:750) "AsyncConn Chore Service.Chore.1" #533746 daemon prio=5 os_prio=0 tid=0x00007fc02ecdc800 nid=0x975 waiting on condition [0x00007fc0c1304000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007fc668719548> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1093) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:750) "RS_LOG_REPLAY_OPS-regionserver/cod-1e3xsc06g8mjm-worker124:16020-0" #533452 daemon prio=5 os_prio=0 tid=0x00007fc0b491f000 nid=0x5e4 waiting on condition [0x00007fc0c3e2f000] java.lang.Thread.State: WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007fc1dc360c30> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:750) "policyDownloadTimer" #344447 daemon prio=5 os_prio=0 tid=0x0000558b3074d000 nid=0x796b in Object.wait() [0x00007fc0d45cf000] java.lang.Thread.State: TIMED_WAITING (on object monitor) at java.lang.Object.wait(Native Method) at java.util.TimerThread.mainLoop(Timer.java:552) - locked <0x00007fc4e7eacbd8> (a java.util.TaskQueue) at java.util.TimerThread.run(Timer.java:505) "RangerTagRefresher(serviceName=cod_exait3-cod-prod_hbase)-344444" #344444 daemon prio=5 os_prio=0 tid=0x0000558b30744000 nid=0x7969 waiting on condition [0x00007fc0d6dca000] java.lang.Thread.State: WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007fc4e7eacdc0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) at org.apache.ranger.plugin.contextenricher.RangerTagEnricher$RangerTagRefresher.run(RangerTagEnricher.java:949) "IPC Parameter Sending Thread #2" #296848 daemon prio=5 os_prio=0 tid=0x0000558b221ab800 nid=0x4ee5 waiting on condition [0x00007fc0c2617000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007fc1c9d2acb8> (a java.util.concurrent.SynchronousQueue$TransferStack) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460) at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362) at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:750) "Attach Listener" #240139 daemon prio=9 os_prio=0 tid=0x00007fc03d9c9800 nid=0x3ab7 waiting on condition [0x0000000000000000] java.lang.Thread.State: RUNNABLE "ReadOnlyZKClient-cod-1e3xsc06g8mjm-master1.exait3p.z1did1.b0.cloudera.site:2181,cod-1e3xsc06g8mjm-master0.exait3p.z1did1.b0.cloudera.site:2181,cod-1e3xsc06g8mjm-leader0.exait3p.z1did1.b0.cloudera.site:2181@0x679c67d4" #229591 daemon prio=5 os_prio=0 tid=0x00007fc0b227c800 nid=0x6ad1 waiting on condition [0x00007fc0c2e1f000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007fc415ac41f0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.DelayQueue.poll(DelayQueue.java:259) at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:323) at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$$Lambda$164/1053864765.run(Unknown Source) at java.lang.Thread.run(Thread.java:750) "AsyncConn Chore Service.Chore.1" #229590 daemon prio=5 os_prio=0 tid=0x00007fc08e32a800 nid=0x6ad0 waiting on condition [0x00007fc0e9052000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007fc415ac7e50> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1093) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:750) "ReadOnlyZKClient-cod-1e3xsc06g8mjm-master1.exait3p.z1did1.b0.cloudera.site:2181,cod-1e3xsc06g8mjm-master0.exait3p.z1did1.b0.cloudera.site:2181,cod-1e3xsc06g8mjm-leader0.exait3p.z1did1.b0.cloudera.site:2181@0x338a1ba7" #175416 daemon prio=5 os_prio=0 tid=0x00007fc0ad801800 nid=0x5fce waiting on condition [0x00007fc0c5cdb000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007fc297f20978> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.DelayQueue.poll(DelayQueue.java:259) at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:323) at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$$Lambda$164/1053864765.run(Unknown Source) at java.lang.Thread.run(Thread.java:750) "AsyncConn Chore Service.Chore.1" #175415 daemon prio=5 os_prio=0 tid=0x00007fc08443c800 nid=0x5fcd waiting on condition [0x00007fc0dffc2000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007fc297f28bd0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1093) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:750) "ReadOnlyZKClient-cod-1e3xsc06g8mjm-master1.exait3p.z1did1.b0.cloudera.site:2181,cod-1e3xsc06g8mjm-master0.exait3p.z1did1.b0.cloudera.site:2181,cod-1e3xsc06g8mjm-leader0.exait3p.z1did1.b0.cloudera.site:2181@0x7032b968" #175383 daemon prio=5 os_prio=0 tid=0x0000558b2a42e000 nid=0x5f75 waiting on condition [0x00007fc0f05c5000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007fc297b5e178> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.DelayQueue.poll(DelayQueue.java:259) at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:323) at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$$Lambda$164/1053864765.run(Unknown Source) at java.lang.Thread.run(Thread.java:750) "AsyncConn Chore Service.Chore.1" #175382 daemon prio=5 os_prio=0 tid=0x0000558b21fa3000 nid=0x5f74 waiting on condition [0x00007fc0c6fee000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007fc297b60178> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1093) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:750) "ReadOnlyZKClient-cod-1e3xsc06g8mjm-master1.exait3p.z1did1.b0.cloudera.site:2181,cod-1e3xsc06g8mjm-master0.exait3p.z1did1.b0.cloudera.site:2181,cod-1e3xsc06g8mjm-leader0.exait3p.z1did1.b0.cloudera.site:2181@0x20cac4b9" #175092 daemon prio=5 os_prio=0 tid=0x00007fc05d2d0000 nid=0x5b17 waiting on condition [0x00007fc0d6ecb000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007fc297b603b8> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.DelayQueue.poll(DelayQueue.java:259) at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:323) at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$$Lambda$164/1053864765.run(Unknown Source) at java.lang.Thread.run(Thread.java:750) "AsyncConn Chore Service.Chore.1" #175091 daemon prio=5 os_prio=0 tid=0x00007fc11cfdd000 nid=0x5b16 waiting on condition [0x00007fc0d70cd000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007fc297b5e378> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1093) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:750) "ReadOnlyZKClient-cod-1e3xsc06g8mjm-master1.exait3p.z1did1.b0.cloudera.site:2181,cod-1e3xsc06g8mjm-master0.exait3p.z1did1.b0.cloudera.site:2181,cod-1e3xsc06g8mjm-leader0.exait3p.z1did1.b0.cloudera.site:2181@0x7b3ad8be" #173712 daemon prio=5 os_prio=0 tid=0x0000558b21c8d000 nid=0x470e waiting on condition [0x00007fc0e00c3000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007fc280f90f78> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.DelayQueue.poll(DelayQueue.java:259) at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:323) at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$$Lambda$164/1053864765.run(Unknown Source) at java.lang.Thread.run(Thread.java:750) "AsyncConn Chore Service.Chore.1" #173711 daemon prio=5 os_prio=0 tid=0x0000558b20454000 nid=0x46f6 waiting on condition [0x00007fc0c6dec000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007fc280f96f78> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1093) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:750) "ReadOnlyZKClient-cod-1e3xsc06g8mjm-master1.exait3p.z1did1.b0.cloudera.site:2181,cod-1e3xsc06g8mjm-master0.exait3p.z1did1.b0.cloudera.site:2181,cod-1e3xsc06g8mjm-leader0.exait3p.z1did1.b0.cloudera.site:2181@0x7a7a7246" #173546 daemon prio=5 os_prio=0 tid=0x00007fc0adc38000 nid=0x44c1 waiting on condition [0x00007fc0e7032000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007fc280f94f78> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.DelayQueue.poll(DelayQueue.java:259) at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:323) at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$$Lambda$164/1053864765.run(Unknown Source) at java.lang.Thread.run(Thread.java:750) "AsyncConn Chore Service.Chore.1" #173545 daemon prio=5 os_prio=0 tid=0x00007fc09e5f2800 nid=0x44c0 waiting on condition [0x00007fc0d6cc9000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007fc280f92f78> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1093) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:750) "ReadOnlyZKClient-cod-1e3xsc06g8mjm-master1.exait3p.z1did1.b0.cloudera.site:2181,cod-1e3xsc06g8mjm-master0.exait3p.z1did1.b0.cloudera.site:2181,cod-1e3xsc06g8mjm-leader0.exait3p.z1did1.b0.cloudera.site:2181@0x562855f6" #172293 daemon prio=5 os_prio=0 tid=0x00007fc0aac74000 nid=0x332b waiting on condition [0x00007fc0f6928000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007fc415ac5ca8> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.DelayQueue.poll(DelayQueue.java:259) at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:323) at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$$Lambda$164/1053864765.run(Unknown Source) at java.lang.Thread.run(Thread.java:750) "ReadOnlyZKClient-cod-1e3xsc06g8mjm-master1.exait3p.z1did1.b0.cloudera.site:2181,cod-1e3xsc06g8mjm-master0.exait3p.z1did1.b0.cloudera.site:2181,cod-1e3xsc06g8mjm-leader0.exait3p.z1did1.b0.cloudera.site:2181@0x1fef9018" #172292 daemon prio=5 os_prio=0 tid=0x00007fc11ddc0800 nid=0x332a waiting on condition [0x00007fc0c7efd000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007fc415ac9e50> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.DelayQueue.poll(DelayQueue.java:259) at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:323) at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$$Lambda$164/1053864765.run(Unknown Source) at java.lang.Thread.run(Thread.java:750) "AsyncConn Chore Service.Chore.1" #172291 daemon prio=5 os_prio=0 tid=0x00007fc06f6af000 nid=0x3329 waiting on condition [0x00007fc0e8244000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007fc415ac5ea8> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1093) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:750) "AsyncConn Chore Service.Chore.1" #172290 daemon prio=5 os_prio=0 tid=0x00007fc086612000 nid=0x3328 waiting on condition [0x00007fc0c504c000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007fc415ac60e8> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1093) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:750) "ReadOnlyZKClient-cod-1e3xsc06g8mjm-master1.exait3p.z1did1.b0.cloudera.site:2181,cod-1e3xsc06g8mjm-master0.exait3p.z1did1.b0.cloudera.site:2181,cod-1e3xsc06g8mjm-leader0.exait3p.z1did1.b0.cloudera.site:2181@0x6693f11a" #171172 daemon prio=5 os_prio=0 tid=0x00007fc11d025800 nid=0x241c waiting on condition [0x00007fc0ce0e1000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007fc415ac43f0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.DelayQueue.poll(DelayQueue.java:259) at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:323) at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$$Lambda$164/1053864765.run(Unknown Source) at java.lang.Thread.run(Thread.java:750) "AsyncConn Chore Service.Chore.1" #171171 daemon prio=5 os_prio=0 tid=0x00007fc11cf6c000 nid=0x241b waiting on condition [0x00007fc0c2b1c000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007fc415ac6328> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1093) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:750) "ReadOnlyZKClient-cod-1e3xsc06g8mjm-master1.exait3p.z1did1.b0.cloudera.site:2181,cod-1e3xsc06g8mjm-master0.exait3p.z1did1.b0.cloudera.site:2181,cod-1e3xsc06g8mjm-leader0.exait3p.z1did1.b0.cloudera.site:2181@0x3006f1d6" #171168 daemon prio=5 os_prio=0 tid=0x0000558b27c4e800 nid=0x2418 waiting on condition [0x00007fc0c2d1e000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007fc415ac45f0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.DelayQueue.poll(DelayQueue.java:259) at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:323) at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$$Lambda$164/1053864765.run(Unknown Source) at java.lang.Thread.run(Thread.java:750) "AsyncConn Chore Service.Chore.1" #171167 daemon prio=5 os_prio=0 tid=0x0000558b20379000 nid=0x2417 waiting on condition [0x00007fc0c5854000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007fc415ac8090> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1093) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:750) "ReadOnlyZKClient-cod-1e3xsc06g8mjm-master1.exait3p.z1did1.b0.cloudera.site:2181,cod-1e3xsc06g8mjm-master0.exait3p.z1did1.b0.cloudera.site:2181,cod-1e3xsc06g8mjm-leader0.exait3p.z1did1.b0.cloudera.site:2181@0x545aacbe" #169399 daemon prio=5 os_prio=0 tid=0x00007fc09cdd6800 nid=0xdce waiting on condition [0x00007fc0c1f10000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007fc415aca050> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.DelayQueue.poll(DelayQueue.java:259) at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:323) at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$$Lambda$164/1053864765.run(Unknown Source) at java.lang.Thread.run(Thread.java:750) "AsyncConn Chore Service.Chore.1" #169398 daemon prio=5 os_prio=0 tid=0x00007fc9863c9800 nid=0xdcd waiting on condition [0x00007fc0c2011000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007fc415ae3ca8> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1093) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:750) "ReadOnlyZKClient-cod-1e3xsc06g8mjm-master1.exait3p.z1did1.b0.cloudera.site:2181,cod-1e3xsc06g8mjm-master0.exait3p.z1did1.b0.cloudera.site:2181,cod-1e3xsc06g8mjm-leader0.exait3p.z1did1.b0.cloudera.site:2181@0x7e58157f" #169364 daemon prio=5 os_prio=0 tid=0x00007fc08909f800 nid=0xd92 waiting on condition [0x00007fc0c3122000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007fc415ac6568> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.DelayQueue.poll(DelayQueue.java:259) at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:323) at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$$Lambda$164/1053864765.run(Unknown Source) at java.lang.Thread.run(Thread.java:750) "AsyncConn Chore Service.Chore.1" #169363 daemon prio=5 os_prio=0 tid=0x00007fc08bc88800 nid=0xd91 waiting on condition [0x00007fc0c3223000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007fc415ac47f0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1093) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:750) "ReadOnlyZKClient-cod-1e3xsc06g8mjm-master1.exait3p.z1did1.b0.cloudera.site:2181,cod-1e3xsc06g8mjm-master0.exait3p.z1did1.b0.cloudera.site:2181,cod-1e3xsc06g8mjm-leader0.exait3p.z1did1.b0.cloudera.site:2181@0x6636d720" #169347 daemon prio=5 os_prio=0 tid=0x00007fc11d5b0800 nid=0xd69 waiting on condition [0x00007fc0cc916000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007fc415acdca8> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.DelayQueue.poll(DelayQueue.java:259) at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:323) at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$$Lambda$164/1053864765.run(Unknown Source) at java.lang.Thread.run(Thread.java:750) "AsyncConn Chore Service.Chore.1" #169346 daemon prio=5 os_prio=0 tid=0x00007fc05cf44000 nid=0xd68 waiting on condition [0x00007fc0d73d0000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007fc415ae3ee8> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1093) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:750) "ReadOnlyZKClient-cod-1e3xsc06g8mjm-master1.exait3p.z1did1.b0.cloudera.site:2181,cod-1e3xsc06g8mjm-master0.exait3p.z1did1.b0.cloudera.site:2181,cod-1e3xsc06g8mjm-leader0.exait3p.z1did1.b0.cloudera.site:2181@0x678f20d0" #169307 daemon prio=5 os_prio=0 tid=0x00007fc09e26f000 nid=0xd2b waiting on condition [0x00007fc0ce6e9000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007fc6be88a378> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.DelayQueue.poll(DelayQueue.java:259) at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:323) at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$$Lambda$164/1053864765.run(Unknown Source) at java.lang.Thread.run(Thread.java:750) "ReadOnlyZKClient-cod-1e3xsc06g8mjm-master1.exait3p.z1did1.b0.cloudera.site:2181,cod-1e3xsc06g8mjm-master0.exait3p.z1did1.b0.cloudera.site:2181,cod-1e3xsc06g8mjm-leader0.exait3p.z1did1.b0.cloudera.site:2181@0x20d25f64" #169306 daemon prio=5 os_prio=0 tid=0x00007fc089dcb000 nid=0xd2a waiting on condition [0x00007fc0ceeef000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007fc6be888378> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.DelayQueue.poll(DelayQueue.java:259) at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:323) at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$$Lambda$164/1053864765.run(Unknown Source) at java.lang.Thread.run(Thread.java:750) "AsyncConn Chore Service.Chore.1" #169305 daemon prio=5 os_prio=0 tid=0x00007fc090a61800 nid=0xd29 waiting on condition [0x00007fc0cf2f3000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007fc6be886378> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1093) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:750) "AsyncConn Chore Service.Chore.1" #169304 daemon prio=5 os_prio=0 tid=0x00007fc97d036000 nid=0xd28 waiting on condition [0x00007fc0c4d49000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007fc6be88a578> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1093) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:750) "ReadOnlyZKClient-cod-1e3xsc06g8mjm-master1.exait3p.z1did1.b0.cloudera.site:2181,cod-1e3xsc06g8mjm-master0.exait3p.z1did1.b0.cloudera.site:2181,cod-1e3xsc06g8mjm-leader0.exait3p.z1did1.b0.cloudera.site:2181@0x1ff82248" #169295 daemon prio=5 os_prio=0 tid=0x0000558b17608000 nid=0xd1d waiting on condition [0x00007fc0c5551000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007fc6be888578> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.DelayQueue.poll(DelayQueue.java:259) at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:323) at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$$Lambda$164/1053864765.run(Unknown Source) at java.lang.Thread.run(Thread.java:750) "AsyncConn Chore Service.Chore.1" #169294 daemon prio=5 os_prio=0 tid=0x0000558b1446a800 nid=0xd1c waiting on condition [0x00007fc0d44ce000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007fc6be8865b8> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1093) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:750) "ReadOnlyZKClient-cod-1e3xsc06g8mjm-master1.exait3p.z1did1.b0.cloudera.site:2181,cod-1e3xsc06g8mjm-master0.exait3p.z1did1.b0.cloudera.site:2181,cod-1e3xsc06g8mjm-leader0.exait3p.z1did1.b0.cloudera.site:2181@0x514c0883" #169274 daemon prio=5 os_prio=0 tid=0x00007fc09f64c000 nid=0xd04 waiting on condition [0x00007fc0e07ca000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007fc6be88a7b8> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.DelayQueue.poll(DelayQueue.java:259) at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:323) at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$$Lambda$164/1053864765.run(Unknown Source) at java.lang.Thread.run(Thread.java:750) "AsyncConn Chore Service.Chore.1" #169273 daemon prio=5 os_prio=0 tid=0x00007fc0af25b800 nid=0xd03 waiting on condition [0x00007fc0e8d4f000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007fc6be888778> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1093) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:750) "ReadOnlyZKClient-cod-1e3xsc06g8mjm-master1.exait3p.z1did1.b0.cloudera.site:2181,cod-1e3xsc06g8mjm-master0.exait3p.z1did1.b0.cloudera.site:2181,cod-1e3xsc06g8mjm-leader0.exait3p.z1did1.b0.cloudera.site:2181@0x25b4a91b" #169263 daemon prio=5 os_prio=0 tid=0x00007fc06ff5f000 nid=0xcf8 waiting on condition [0x00007fc0e3e01000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007fc6be8867f8> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.DelayQueue.poll(DelayQueue.java:259) at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:323) at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$$Lambda$164/1053864765.run(Unknown Source) at java.lang.Thread.run(Thread.java:750) "AsyncConn Chore Service.Chore.1" #169262 daemon prio=5 os_prio=0 tid=0x00007fc06cc5c800 nid=0xcf7 waiting on condition [0x00007fc0e4406000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007fc6be88a9b8> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1093) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:750) "ReadOnlyZKClient-cod-1e3xsc06g8mjm-master1.exait3p.z1did1.b0.cloudera.site:2181,cod-1e3xsc06g8mjm-master0.exait3p.z1did1.b0.cloudera.site:2181,cod-1e3xsc06g8mjm-leader0.exait3p.z1did1.b0.cloudera.site:2181@0x260f19c6" #169242 daemon prio=5 os_prio=0 tid=0x00007fc092121800 nid=0xce1 waiting on condition [0x00007fc0c68e7000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007fc6be88abf8> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.DelayQueue.poll(DelayQueue.java:259) at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:323) at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$$Lambda$164/1053864765.run(Unknown Source) at java.lang.Thread.run(Thread.java:750) "ReadOnlyZKClient-cod-1e3xsc06g8mjm-master1.exait3p.z1did1.b0.cloudera.site:2181,cod-1e3xsc06g8mjm-master0.exait3p.z1did1.b0.cloudera.site:2181,cod-1e3xsc06g8mjm-leader0.exait3p.z1did1.b0.cloudera.site:2181@0x763ebe02" #169241 daemon prio=5 os_prio=0 tid=0x00007fc0a61dd000 nid=0xce0 waiting on condition [0x00007fc0c6eed000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007fc6be8889b8> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.DelayQueue.poll(DelayQueue.java:259) at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:323) at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$$Lambda$164/1053864765.run(Unknown Source) at java.lang.Thread.run(Thread.java:750) "ReadOnlyZKClient-cod-1e3xsc06g8mjm-master1.exait3p.z1did1.b0.cloudera.site:2181,cod-1e3xsc06g8mjm-master0.exait3p.z1did1.b0.cloudera.site:2181,cod-1e3xsc06g8mjm-leader0.exait3p.z1did1.b0.cloudera.site:2181@0x1b1d2127" #169240 daemon prio=5 os_prio=0 tid=0x00007fc093ded800 nid=0xcdf waiting on condition [0x00007fc0cc68f000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007fc6be8869f8> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.DelayQueue.poll(DelayQueue.java:259) at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:323) at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$$Lambda$164/1053864765.run(Unknown Source) at java.lang.Thread.run(Thread.java:750) "AsyncConn Chore Service.Chore.1" #169239 daemon prio=5 os_prio=0 tid=0x00007fc05d388000 nid=0xcde waiting on condition [0x00007fc0c534f000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007fc6be88adf8> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1093) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:750) "AsyncConn Chore Service.Chore.1" #169238 daemon prio=5 os_prio=0 tid=0x00007fc091f26800 nid=0xcdd waiting on condition [0x00007fc0c524e000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007fc6be888bb8> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1093) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:750) "AsyncConn Chore Service.Chore.1" #169237 daemon prio=5 os_prio=0 tid=0x00007fc0d1674800 nid=0xcdc waiting on condition [0x00007fc0c5753000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007fc6be886bf8> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1093) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:750) "ReadOnlyZKClient-cod-1e3xsc06g8mjm-master1.exait3p.z1did1.b0.cloudera.site:2181,cod-1e3xsc06g8mjm-master0.exait3p.z1did1.b0.cloudera.site:2181,cod-1e3xsc06g8mjm-leader0.exait3p.z1did1.b0.cloudera.site:2181@0x64189c62" #169229 daemon prio=5 os_prio=0 tid=0x00007fc08ab70800 nid=0xcd1 waiting on condition [0x00007fc0c5450000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007fc6be88b038> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.DelayQueue.poll(DelayQueue.java:259) at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:323) at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$$Lambda$164/1053864765.run(Unknown Source) at java.lang.Thread.run(Thread.java:750) "AsyncConn Chore Service.Chore.1" #169228 daemon prio=5 os_prio=0 tid=0x00007fc08d48e000 nid=0xcd0 waiting on condition [0x00007fc0c4b47000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007fc6be888df8> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1093) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:750) "ReadOnlyZKClient-cod-1e3xsc06g8mjm-master1.exait3p.z1did1.b0.cloudera.site:2181,cod-1e3xsc06g8mjm-master0.exait3p.z1did1.b0.cloudera.site:2181,cod-1e3xsc06g8mjm-leader0.exait3p.z1did1.b0.cloudera.site:2181@0x1d056d12" #169222 daemon prio=5 os_prio=0 tid=0x00007fc05cf49000 nid=0xcca waiting on condition [0x00007fc0c4031000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007fc6be88b238> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.DelayQueue.poll(DelayQueue.java:259) at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:323) at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$$Lambda$164/1053864765.run(Unknown Source) at java.lang.Thread.run(Thread.java:750) "AsyncConn Chore Service.Chore.1" #169221 daemon prio=5 os_prio=0 tid=0x00007fc11d7bb000 nid=0xcc9 waiting on condition [0x00007fc0c4440000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007fc6be886e38> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1093) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:750) "ReadOnlyZKClient-cod-1e3xsc06g8mjm-master1.exait3p.z1did1.b0.cloudera.site:2181,cod-1e3xsc06g8mjm-master0.exait3p.z1did1.b0.cloudera.site:2181,cod-1e3xsc06g8mjm-leader0.exait3p.z1did1.b0.cloudera.site:2181@0x11e58944" #169217 daemon prio=5 os_prio=0 tid=0x0000558b171a1800 nid=0xcc3 waiting on condition [0x00007fc0c3d2e000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007fc6be889038> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.DelayQueue.poll(DelayQueue.java:259) at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:323) at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$$Lambda$164/1053864765.run(Unknown Source) at java.lang.Thread.run(Thread.java:750) "AsyncConn Chore Service.Chore.1" #169216 daemon prio=5 os_prio=0 tid=0x0000558b29abf800 nid=0xcc2 waiting on condition [0x00007fc0c3b2c000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007fc6be887078> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1093) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:750) "ReadOnlyZKClient-cod-1e3xsc06g8mjm-master1.exait3p.z1did1.b0.cloudera.site:2181,cod-1e3xsc06g8mjm-master0.exait3p.z1did1.b0.cloudera.site:2181,cod-1e3xsc06g8mjm-leader0.exait3p.z1did1.b0.cloudera.site:2181@0x35043c28" #168691 daemon prio=5 os_prio=0 tid=0x00007fc09f4bd000 nid=0x4e9 waiting on condition [0x00007fc0c4945000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007fc500252538> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.DelayQueue.poll(DelayQueue.java:259) at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:323) at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$$Lambda$164/1053864765.run(Unknown Source) at java.lang.Thread.run(Thread.java:750) "AsyncConn Chore Service.Chore.1" #168690 daemon prio=5 os_prio=0 tid=0x00007fc075407000 nid=0x4e8 waiting on condition [0x00007fc0c4a46000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007fc500254f78> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1093) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:750) "ReadOnlyZKClient-cod-1e3xsc06g8mjm-master1.exait3p.z1did1.b0.cloudera.site:2181,cod-1e3xsc06g8mjm-master0.exait3p.z1did1.b0.cloudera.site:2181,cod-1e3xsc06g8mjm-leader0.exait3p.z1did1.b0.cloudera.site:2181@0x669c9b95" #168675 daemon prio=5 os_prio=0 tid=0x0000558b27c02000 nid=0x4d7 waiting on condition [0x00007fc0c5edd000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007fc500258f78> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.DelayQueue.poll(DelayQueue.java:259) at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:323) at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$$Lambda$164/1053864765.run(Unknown Source) at java.lang.Thread.run(Thread.java:750) "ReadOnlyZKClient-cod-1e3xsc06g8mjm-master1.exait3p.z1did1.b0.cloudera.site:2181,cod-1e3xsc06g8mjm-master0.exait3p.z1did1.b0.cloudera.site:2181,cod-1e3xsc06g8mjm-leader0.exait3p.z1did1.b0.cloudera.site:2181@0x1a57247f" #168674 daemon prio=5 os_prio=0 tid=0x00007fc0a54cc800 nid=0x4d6 waiting on condition [0x00007fc0c60df000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007fc500252738> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.DelayQueue.poll(DelayQueue.java:259) at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:323) at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$$Lambda$164/1053864765.run(Unknown Source) at java.lang.Thread.run(Thread.java:750) "AsyncConn Chore Service.Chore.1" #168673 daemon prio=5 os_prio=0 tid=0x0000558b1d9bb000 nid=0x4d5 waiting on condition [0x00007fc0c62e1000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007fc5002551b8> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1093) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:750) "AsyncConn Chore Service.Chore.1" #168672 daemon prio=5 os_prio=0 tid=0x00007fc05c62a000 nid=0x4d4 waiting on condition [0x00007fc0c65e4000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007fc500259178> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1093) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:750) "ReadOnlyZKClient-cod-1e3xsc06g8mjm-master1.exait3p.z1did1.b0.cloudera.site:2181,cod-1e3xsc06g8mjm-master0.exait3p.z1did1.b0.cloudera.site:2181,cod-1e3xsc06g8mjm-leader0.exait3p.z1did1.b0.cloudera.site:2181@0x4c59fc37" #168648 daemon prio=5 os_prio=0 tid=0x00007fc0c927d000 nid=0x4b3 waiting on condition [0x00007fc0c78f7000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007fc500252938> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.DelayQueue.poll(DelayQueue.java:259) at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:323) at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$$Lambda$164/1053864765.run(Unknown Source) at java.lang.Thread.run(Thread.java:750) "AsyncConn Chore Service.Chore.1" #168647 daemon prio=5 os_prio=0 tid=0x00007fc07c763800 nid=0x4b2 waiting on condition [0x00007fc0cc18a000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007fc5002553f8> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1093) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:750) "ReadOnlyZKClient-cod-1e3xsc06g8mjm-master1.exait3p.z1did1.b0.cloudera.site:2181,cod-1e3xsc06g8mjm-master0.exait3p.z1did1.b0.cloudera.site:2181,cod-1e3xsc06g8mjm-leader0.exait3p.z1did1.b0.cloudera.site:2181@0x6c4e6cf2" #168617 daemon prio=5 os_prio=0 tid=0x00007fc11c99c800 nid=0x487 waiting on condition [0x00007fc0ce1e2000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007fc5002593b8> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.DelayQueue.poll(DelayQueue.java:259) at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:323) at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$$Lambda$164/1053864765.run(Unknown Source) at java.lang.Thread.run(Thread.java:750) "AsyncConn Chore Service.Chore.1" #168616 daemon prio=5 os_prio=0 tid=0x00007fc11ddb1000 nid=0x486 waiting on condition [0x00007fc0cf4f5000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007fc500255638> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1093) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:750) "ReadOnlyZKClient-cod-1e3xsc06g8mjm-master1.exait3p.z1did1.b0.cloudera.site:2181,cod-1e3xsc06g8mjm-master0.exait3p.z1did1.b0.cloudera.site:2181,cod-1e3xsc06g8mjm-leader0.exait3p.z1did1.b0.cloudera.site:2181@0x132a144a" #168602 daemon prio=5 os_prio=0 tid=0x00007fc087a80000 nid=0x473 waiting on condition [0x00007fc0d46d0000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007fc5002595b8> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.DelayQueue.poll(DelayQueue.java:259) at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:323) at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$$Lambda$164/1053864765.run(Unknown Source) at java.lang.Thread.run(Thread.java:750) "AsyncConn Chore Service.Chore.1" #168601 daemon prio=5 os_prio=0 tid=0x00007fc06b347800 nid=0x472 waiting on condition [0x00007fc0d6bc8000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007fc500255878> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1093) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:750) "ReadOnlyZKClient-cod-1e3xsc06g8mjm-master1.exait3p.z1did1.b0.cloudera.site:2181,cod-1e3xsc06g8mjm-master0.exait3p.z1did1.b0.cloudera.site:2181,cod-1e3xsc06g8mjm-leader0.exait3p.z1did1.b0.cloudera.site:2181@0x452dc0a4" #168598 daemon prio=5 os_prio=0 tid=0x00007fc11dd9d000 nid=0x46e waiting on condition [0x00007fc0d72cf000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007fc5002597b8> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.DelayQueue.poll(DelayQueue.java:259) at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:323) at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$$Lambda$164/1053864765.run(Unknown Source) at java.lang.Thread.run(Thread.java:750) "AsyncConn Chore Service.Chore.1" #168597 daemon prio=5 os_prio=0 tid=0x00007fc11d015800 nid=0x46d waiting on condition [0x00007fc0d77d4000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007fc5002599b8> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1093) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:750) "ReadOnlyZKClient-cod-1e3xsc06g8mjm-master1.exait3p.z1did1.b0.cloudera.site:2181,cod-1e3xsc06g8mjm-master0.exait3p.z1did1.b0.cloudera.site:2181,cod-1e3xsc06g8mjm-leader0.exait3p.z1did1.b0.cloudera.site:2181@0x343ad634" #168587 daemon prio=5 os_prio=0 tid=0x00007fc0c81ce800 nid=0x462 waiting on condition [0x00007fc0e19dc000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007fc500255ab8> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.DelayQueue.poll(DelayQueue.java:259) at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:323) at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$$Lambda$164/1053864765.run(Unknown Source) at java.lang.Thread.run(Thread.java:750) "AsyncConn Chore Service.Chore.1" #168586 daemon prio=5 os_prio=0 tid=0x00007fc08ade3000 nid=0x461 waiting on condition [0x00007fc0e6b2d000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007fc500259bf8> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1093) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:750) "ReadOnlyZKClient-cod-1e3xsc06g8mjm-master1.exait3p.z1did1.b0.cloudera.site:2181,cod-1e3xsc06g8mjm-master0.exait3p.z1did1.b0.cloudera.site:2181,cod-1e3xsc06g8mjm-leader0.exait3p.z1did1.b0.cloudera.site:2181@0x2d9f9a02" #168582 daemon prio=5 os_prio=0 tid=0x00007fc09fff4000 nid=0x443 waiting on condition [0x00007fc0e35f8000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007fc500255cb8> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.DelayQueue.poll(DelayQueue.java:259) at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:323) at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$$Lambda$164/1053864765.run(Unknown Source) at java.lang.Thread.run(Thread.java:750) "AsyncConn Chore Service.Chore.1" #168581 daemon prio=5 os_prio=0 tid=0x00007fc08459e800 nid=0x442 waiting on condition [0x00007fc0e3bff000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007fc500259e38> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1093) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:750) "ReadOnlyZKClient-cod-1e3xsc06g8mjm-master1.exait3p.z1did1.b0.cloudera.site:2181,cod-1e3xsc06g8mjm-master0.exait3p.z1did1.b0.cloudera.site:2181,cod-1e3xsc06g8mjm-leader0.exait3p.z1did1.b0.cloudera.site:2181@0x01e48bca" #168565 daemon prio=5 os_prio=0 tid=0x0000558b207c9000 nid=0x42d waiting on condition [0x00007fc0e9d5f000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007fc50024af78> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.DelayQueue.poll(DelayQueue.java:259) at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:323) at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$$Lambda$164/1053864765.run(Unknown Source) at java.lang.Thread.run(Thread.java:750) "AsyncConn Chore Service.Chore.1" #168564 daemon prio=5 os_prio=0 tid=0x0000558b275f2000 nid=0x42c waiting on condition [0x00007fc0ead6f000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007fc500255eb8> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1093) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:750) "ReadOnlyZKClient-cod-1e3xsc06g8mjm-master1.exait3p.z1did1.b0.cloudera.site:2181,cod-1e3xsc06g8mjm-master0.exait3p.z1did1.b0.cloudera.site:2181,cod-1e3xsc06g8mjm-leader0.exait3p.z1did1.b0.cloudera.site:2181@0x0c398bbf" #168555 daemon prio=5 os_prio=0 tid=0x00007fc11d027000 nid=0x41f waiting on condition [0x00007fc1012cd000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007fc50025a078> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.DelayQueue.poll(DelayQueue.java:259) at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:323) at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$$Lambda$164/1053864765.run(Unknown Source) at java.lang.Thread.run(Thread.java:750) "AsyncConn Chore Service.Chore.1" #168554 daemon prio=5 os_prio=0 tid=0x00007fc0a7348800 nid=0x41e waiting on condition [0x00007fc109222000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007fc5002560f8> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1093) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:750) "ReadOnlyZKClient-cod-1e3xsc06g8mjm-master1.exait3p.z1did1.b0.cloudera.site:2181,cod-1e3xsc06g8mjm-master0.exait3p.z1did1.b0.cloudera.site:2181,cod-1e3xsc06g8mjm-leader0.exait3p.z1did1.b0.cloudera.site:2181@0x52170f4e" #168545 daemon prio=5 os_prio=0 tid=0x00007fc09feb9800 nid=0x415 waiting on condition [0x00007fc0c6ceb000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007fc50025a278> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.DelayQueue.poll(DelayQueue.java:259) at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:323) at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$$Lambda$164/1053864765.run(Unknown Source) at java.lang.Thread.run(Thread.java:750) "ReadOnlyZKClient-cod-1e3xsc06g8mjm-master1.exait3p.z1did1.b0.cloudera.site:2181,cod-1e3xsc06g8mjm-master0.exait3p.z1did1.b0.cloudera.site:2181,cod-1e3xsc06g8mjm-leader0.exait3p.z1did1.b0.cloudera.site:2181@0x2b24bbe1" #168544 daemon prio=5 os_prio=0 tid=0x00007fc0844da000 nid=0x414 waiting on condition [0x00007fc0c514d000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007fc50024b178> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.DelayQueue.poll(DelayQueue.java:259) at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:323) at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$$Lambda$164/1053864765.run(Unknown Source) at java.lang.Thread.run(Thread.java:750) "AsyncConn Chore Service.Chore.1" #168543 daemon prio=5 os_prio=0 tid=0x00007fc092741800 nid=0x413 waiting on condition [0x00007fc0e4507000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007fc500256338> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1093) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:750) "AsyncConn Chore Service.Chore.1" #168542 daemon prio=5 os_prio=0 tid=0x00007fc090502000 nid=0x412 waiting on condition [0x00007fc0c7ffe000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007fc50025a478> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1093) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:750) "ReadOnlyZKClient-cod-1e3xsc06g8mjm-master1.exait3p.z1did1.b0.cloudera.site:2181,cod-1e3xsc06g8mjm-master0.exait3p.z1did1.b0.cloudera.site:2181,cod-1e3xsc06g8mjm-leader0.exait3p.z1did1.b0.cloudera.site:2181@0x593be290" #168539 daemon prio=5 os_prio=0 tid=0x00007fc09f7ab000 nid=0x40f waiting on condition [0x00007fc0d6fcc000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007fc50025a6b8> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.DelayQueue.poll(DelayQueue.java:259) at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:323) at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$$Lambda$164/1053864765.run(Unknown Source) at java.lang.Thread.run(Thread.java:750) "AsyncConn Chore Service.Chore.1" #168538 daemon prio=5 os_prio=0 tid=0x00007fc0900c8800 nid=0x40e waiting on condition [0x00007fc0e37fa000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007fc50023ef90> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1093) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:750) "ReadOnlyZKClient-cod-1e3xsc06g8mjm-master1.exait3p.z1did1.b0.cloudera.site:2181,cod-1e3xsc06g8mjm-master0.exait3p.z1did1.b0.cloudera.site:2181,cod-1e3xsc06g8mjm-leader0.exait3p.z1did1.b0.cloudera.site:2181@0x68c0f300" #168300 daemon prio=5 os_prio=0 tid=0x00007fc11e2bf000 nid=0x7f08 waiting on condition [0x00007fc0e1add000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007fc50025a8b8> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.DelayQueue.poll(DelayQueue.java:259) at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:323) at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$$Lambda$164/1053864765.run(Unknown Source) at java.lang.Thread.run(Thread.java:750) "AsyncConn Chore Service.Chore.1" #168299 daemon prio=5 os_prio=0 tid=0x00007fc11d9e0000 nid=0x7f07 waiting on condition [0x00007fc0e985a000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007fc50025aab8> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1093) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:750) "ReadOnlyZKClient-cod-1e3xsc06g8mjm-master1.exait3p.z1did1.b0.cloudera.site:2181,cod-1e3xsc06g8mjm-master0.exait3p.z1did1.b0.cloudera.site:2181,cod-1e3xsc06g8mjm-leader0.exait3p.z1did1.b0.cloudera.site:2181@0x7b6ab889" #167895 daemon prio=5 os_prio=0 tid=0x00007fc97c06e000 nid=0x7914 waiting on condition [0x00007fc0e02c5000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007fc415ac82d0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.DelayQueue.poll(DelayQueue.java:259) at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:323) at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$$Lambda$164/1053864765.run(Unknown Source) at java.lang.Thread.run(Thread.java:750) "AsyncConn Chore Service.Chore.1" #167894 daemon prio=5 os_prio=0 tid=0x00007fc07c53d000 nid=0x7913 waiting on condition [0x00007fc0e34f7000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007fc415a95178> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1093) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:750) "ReadOnlyZKClient-cod-1e3xsc06g8mjm-master1.exait3p.z1did1.b0.cloudera.site:2181,cod-1e3xsc06g8mjm-master0.exait3p.z1did1.b0.cloudera.site:2181,cod-1e3xsc06g8mjm-leader0.exait3p.z1did1.b0.cloudera.site:2181@0x6386cb87" #167700 daemon prio=5 os_prio=0 tid=0x00007fc11d61e800 nid=0x7677 waiting on condition [0x00007fc0c79f8000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007fc415ac84d0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.DelayQueue.poll(DelayQueue.java:259) at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:323) at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$$Lambda$164/1053864765.run(Unknown Source) at java.lang.Thread.run(Thread.java:750) "AsyncConn Chore Service.Chore.1" #167699 daemon prio=5 os_prio=0 tid=0x00007fc11e530000 nid=0x7676 waiting on condition [0x00007fc0efec0000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007fc415a953b8> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1093) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:750) "ReadOnlyZKClient-cod-1e3xsc06g8mjm-master1.exait3p.z1did1.b0.cloudera.site:2181,cod-1e3xsc06g8mjm-master0.exait3p.z1did1.b0.cloudera.site:2181,cod-1e3xsc06g8mjm-leader0.exait3p.z1did1.b0.cloudera.site:2181@0x5f02b182" #167240 daemon prio=5 os_prio=0 tid=0x00007fc05d554000 nid=0x6f71 waiting on condition [0x00007fc0e8042000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007fc6fe0487f0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.DelayQueue.poll(DelayQueue.java:259) at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:323) at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$$Lambda$164/1053864765.run(Unknown Source) at java.lang.Thread.run(Thread.java:750) "AsyncConn Chore Service.Chore.1" #167239 daemon prio=5 os_prio=0 tid=0x00007fc05cfa9800 nid=0x6f70 waiting on condition [0x00007fc0f26e6000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007fc6fe0489f0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1093) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:750) "ReadOnlyZKClient-cod-1e3xsc06g8mjm-master1.exait3p.z1did1.b0.cloudera.site:2181,cod-1e3xsc06g8mjm-master0.exait3p.z1did1.b0.cloudera.site:2181,cod-1e3xsc06g8mjm-leader0.exait3p.z1did1.b0.cloudera.site:2181@0x6ddbc8e5" #166980 daemon prio=5 os_prio=0 tid=0x0000558b27c13800 nid=0x6bbd waiting on condition [0x00007fc0ceaeb000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007fc6fe048c30> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.DelayQueue.poll(DelayQueue.java:259) at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:323) at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$$Lambda$164/1053864765.run(Unknown Source) at java.lang.Thread.run(Thread.java:750) "AsyncConn Chore Service.Chore.1" #166979 daemon prio=5 os_prio=0 tid=0x0000558b1de92000 nid=0x6bbc waiting on condition [0x00007fc0ce3e4000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007fc6fe048e30> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1093) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:750) "ReadOnlyZKClient-cod-1e3xsc06g8mjm-master1.exait3p.z1did1.b0.cloudera.site:2181,cod-1e3xsc06g8mjm-master0.exait3p.z1did1.b0.cloudera.site:2181,cod-1e3xsc06g8mjm-leader0.exait3p.z1did1.b0.cloudera.site:2181@0x2ee18942" #166561 daemon prio=5 os_prio=0 tid=0x00007fc0a6f46800 nid=0x6633 waiting on condition [0x00007fc0e8345000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007fc6fe049090> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.DelayQueue.poll(DelayQueue.java:259) at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:323) at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$$Lambda$164/1053864765.run(Unknown Source) at java.lang.Thread.run(Thread.java:750) "AsyncConn Chore Service.Chore.1" #166560 daemon prio=5 os_prio=0 tid=0x00007fc11d4e2000 nid=0x6632 waiting on condition [0x00007fc0cf6f7000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007fc6fe049290> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1093) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:750) "ReadOnlyZKClient-cod-1e3xsc06g8mjm-master1.exait3p.z1did1.b0.cloudera.site:2181,cod-1e3xsc06g8mjm-master0.exait3p.z1did1.b0.cloudera.site:2181,cod-1e3xsc06g8mjm-leader0.exait3p.z1did1.b0.cloudera.site:2181@0x49e032f1" #166538 daemon prio=5 os_prio=0 tid=0x00007fc0961e9000 nid=0x6610 waiting on condition [0x00007fc0e8749000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007fc6fe0494d0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.DelayQueue.poll(DelayQueue.java:259) at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:323) at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$$Lambda$164/1053864765.run(Unknown Source) at java.lang.Thread.run(Thread.java:750) "AsyncConn Chore Service.Chore.1" #166537 daemon prio=5 os_prio=0 tid=0x00007fc97d0e1000 nid=0x660f waiting on condition [0x00007fc0e9759000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007fc6fe0496d0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1093) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:750) "ReadOnlyZKClient-cod-1e3xsc06g8mjm-master1.exait3p.z1did1.b0.cloudera.site:2181,cod-1e3xsc06g8mjm-master0.exait3p.z1did1.b0.cloudera.site:2181,cod-1e3xsc06g8mjm-leader0.exait3p.z1did1.b0.cloudera.site:2181@0x16d6b4cd" #166524 daemon prio=5 os_prio=0 tid=0x00007fc0924cc000 nid=0x65fa waiting on condition [0x00007fc0c6ae9000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007fc6fe049910> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.DelayQueue.poll(DelayQueue.java:259) at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:323) at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$$Lambda$164/1053864765.run(Unknown Source) at java.lang.Thread.run(Thread.java:750) "AsyncConn Chore Service.Chore.1" #166523 daemon prio=5 os_prio=0 tid=0x00007fc09189b800 nid=0x65f9 waiting on condition [0x00007fc0c63e2000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007fc6fe049b10> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1093) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:750) "ReadOnlyZKClient-cod-1e3xsc06g8mjm-master1.exait3p.z1did1.b0.cloudera.site:2181,cod-1e3xsc06g8mjm-master0.exait3p.z1did1.b0.cloudera.site:2181,cod-1e3xsc06g8mjm-leader0.exait3p.z1did1.b0.cloudera.site:2181@0x7b0b500a" #166506 daemon prio=5 os_prio=0 tid=0x00007fc0b5b38800 nid=0x65b3 waiting on condition [0x00007fc0c70ef000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007fc6fe049d50> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.DelayQueue.poll(DelayQueue.java:259) at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:323) at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$$Lambda$164/1053864765.run(Unknown Source) at java.lang.Thread.run(Thread.java:750) "AsyncConn Chore Service.Chore.1" #166505 daemon prio=5 os_prio=0 tid=0x00007fc0a5901000 nid=0x65b2 waiting on condition [0x00007fc0d42cc000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007fc6fe049f70> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1093) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:750) "ReadOnlyZKClient-cod-1e3xsc06g8mjm-master1.exait3p.z1did1.b0.cloudera.site:2181,cod-1e3xsc06g8mjm-master0.exait3p.z1did1.b0.cloudera.site:2181,cod-1e3xsc06g8mjm-leader0.exait3p.z1did1.b0.cloudera.site:2181@0x08b18fcb" #166481 daemon prio=5 os_prio=0 tid=0x00007fc97cd37000 nid=0x656c waiting on condition [0x00007fc0cd8d9000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007fc6fe04a1d0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.DelayQueue.poll(DelayQueue.java:259) at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:323) at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$$Lambda$164/1053864765.run(Unknown Source) at java.lang.Thread.run(Thread.java:750) "AsyncConn Chore Service.Chore.1" #166480 daemon prio=5 os_prio=0 tid=0x00007fc095d77000 nid=0x656b waiting on condition [0x00007fc0cd7d8000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007fc6fe0563e8> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1093) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:750) "ReadOnlyZKClient-cod-1e3xsc06g8mjm-master1.exait3p.z1did1.b0.cloudera.site:2181,cod-1e3xsc06g8mjm-master0.exait3p.z1did1.b0.cloudera.site:2181,cod-1e3xsc06g8mjm-leader0.exait3p.z1did1.b0.cloudera.site:2181@0x284751f1" #166460 daemon prio=5 os_prio=0 tid=0x00007fc06f6a6000 nid=0x6551 waiting on condition [0x00007fc0e7a3e000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007fc6fe056628> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.DelayQueue.poll(DelayQueue.java:259) at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:323) at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$$Lambda$164/1053864765.run(Unknown Source) at java.lang.Thread.run(Thread.java:750) "ReadOnlyZKClient-cod-1e3xsc06g8mjm-master1.exait3p.z1did1.b0.cloudera.site:2181,cod-1e3xsc06g8mjm-master0.exait3p.z1did1.b0.cloudera.site:2181,cod-1e3xsc06g8mjm-leader0.exait3p.z1did1.b0.cloudera.site:2181@0x10b9a016" #166459 daemon prio=5 os_prio=0 tid=0x0000558b273f3800 nid=0x6550 waiting on condition [0x00007fc0eaa6c000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007fc6fe056828> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.DelayQueue.poll(DelayQueue.java:259) at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:323) at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$$Lambda$164/1053864765.run(Unknown Source) at java.lang.Thread.run(Thread.java:750) "AsyncConn Chore Service.Chore.1" #166458 daemon prio=5 os_prio=0 tid=0x00007fc06f6e4800 nid=0x654f waiting on condition [0x00007fc0dd5b6000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007fc6fe056a28> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1093) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:750) "AsyncConn Chore Service.Chore.1" #166457 daemon prio=5 os_prio=0 tid=0x0000558b16f11000 nid=0x654d waiting on condition [0x00007fc0c5fde000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007fc6fe056c68> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1093) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:750) "ReadOnlyZKClient-cod-1e3xsc06g8mjm-master1.exait3p.z1did1.b0.cloudera.site:2181,cod-1e3xsc06g8mjm-master0.exait3p.z1did1.b0.cloudera.site:2181,cod-1e3xsc06g8mjm-leader0.exait3p.z1did1.b0.cloudera.site:2181@0x19fda445" #166449 daemon prio=5 os_prio=0 tid=0x00007fc0a60a1800 nid=0x6542 waiting on condition [0x00007fc0e2ff2000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007fc6fe056ea8> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.DelayQueue.poll(DelayQueue.java:259) at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:323) at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$$Lambda$164/1053864765.run(Unknown Source) at java.lang.Thread.run(Thread.java:750) "AsyncConn Chore Service.Chore.1" #166448 daemon prio=5 os_prio=0 tid=0x00007fc0a5875000 nid=0x6541 waiting on condition [0x00007fc0e01c4000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007fc6fe0570a8> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1093) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:750) "ReadOnlyZKClient-cod-1e3xsc06g8mjm-master1.exait3p.z1did1.b0.cloudera.site:2181,cod-1e3xsc06g8mjm-master0.exait3p.z1did1.b0.cloudera.site:2181,cod-1e3xsc06g8mjm-leader0.exait3p.z1did1.b0.cloudera.site:2181@0x1cd082e0" #166435 daemon prio=5 os_prio=0 tid=0x00007fc090d37800 nid=0x652f waiting on condition [0x00007fc0cdcdd000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007fc6fe0572e8> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.DelayQueue.poll(DelayQueue.java:259) at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:323) at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$$Lambda$164/1053864765.run(Unknown Source) at java.lang.Thread.run(Thread.java:750) "AsyncConn Chore Service.Chore.1" #166434 daemon prio=5 os_prio=0 tid=0x00007fc084eac800 nid=0x652e waiting on condition [0x00007fc0e9456000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007fc6fe0574e8> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1093) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:750) "ReadOnlyZKClient-cod-1e3xsc06g8mjm-master1.exait3p.z1did1.b0.cloudera.site:2181,cod-1e3xsc06g8mjm-master0.exait3p.z1did1.b0.cloudera.site:2181,cod-1e3xsc06g8mjm-leader0.exait3p.z1did1.b0.cloudera.site:2181@0x5749cb05" #166392 daemon prio=5 os_prio=0 tid=0x00007fc08cc0e800 nid=0x64e9 waiting on condition [0x00007fc0d41cb000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007fc6fe057728> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.DelayQueue.poll(DelayQueue.java:259) at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:323) at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$$Lambda$164/1053864765.run(Unknown Source) at java.lang.Thread.run(Thread.java:750) "AsyncConn Chore Service.Chore.1" #166391 daemon prio=5 os_prio=0 tid=0x00007fc096acc800 nid=0x64e8 waiting on condition [0x00007fc0cfefd000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007fc6fe057928> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1093) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:750) "ReadOnlyZKClient-cod-1e3xsc06g8mjm-master1.exait3p.z1did1.b0.cloudera.site:2181,cod-1e3xsc06g8mjm-master0.exait3p.z1did1.b0.cloudera.site:2181,cod-1e3xsc06g8mjm-leader0.exait3p.z1did1.b0.cloudera.site:2181@0x592d3b06" #166382 daemon prio=5 os_prio=0 tid=0x00007fc06d499800 nid=0x64c6 waiting on condition [0x00007fc0ce9ea000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007fc6fe057b88> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.DelayQueue.poll(DelayQueue.java:259) at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:323) at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$$Lambda$164/1053864765.run(Unknown Source) at java.lang.Thread.run(Thread.java:750) "AsyncConn Chore Service.Chore.1" #166381 daemon prio=5 os_prio=0 tid=0x00007fc0a5c04800 nid=0x64c5 waiting on condition [0x00007fc0ce4e5000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007fc6fe057d88> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1093) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:750) "ReadOnlyZKClient-cod-1e3xsc06g8mjm-master1.exait3p.z1did1.b0.cloudera.site:2181,cod-1e3xsc06g8mjm-master0.exait3p.z1did1.b0.cloudera.site:2181,cod-1e3xsc06g8mjm-leader0.exait3p.z1did1.b0.cloudera.site:2181@0x06e105f5" #166370 daemon prio=5 os_prio=0 tid=0x00007fc09f8f0800 nid=0x64b6 waiting on condition [0x00007fc0cebec000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007fc6fe057fe8> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.DelayQueue.poll(DelayQueue.java:259) at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:323) at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$$Lambda$164/1053864765.run(Unknown Source) at java.lang.Thread.run(Thread.java:750) "AsyncConn Chore Service.Chore.1" #166369 daemon prio=5 os_prio=0 tid=0x00007fc0aa5bc000 nid=0x64b5 waiting on condition [0x00007fc0cedee000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007fc6fe058200> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1093) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:750) "ReadOnlyZKClient-cod-1e3xsc06g8mjm-master1.exait3p.z1did1.b0.cloudera.site:2181,cod-1e3xsc06g8mjm-master0.exait3p.z1did1.b0.cloudera.site:2181,cod-1e3xsc06g8mjm-leader0.exait3p.z1did1.b0.cloudera.site:2181@0x5d5daef9" #166366 daemon prio=5 os_prio=0 tid=0x00007fc06cc4e000 nid=0x64b2 waiting on condition [0x00007fc0ceced000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007fc6fe058440> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.DelayQueue.poll(DelayQueue.java:259) at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:323) at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$$Lambda$164/1053864765.run(Unknown Source) at java.lang.Thread.run(Thread.java:750) "AsyncConn Chore Service.Chore.1" #166365 daemon prio=5 os_prio=0 tid=0x00007fc0a7c5c000 nid=0x64b1 waiting on condition [0x00007fc0ceff0000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007fc6fe058640> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1093) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:750) "ReadOnlyZKClient-cod-1e3xsc06g8mjm-master1.exait3p.z1did1.b0.cloudera.site:2181,cod-1e3xsc06g8mjm-master0.exait3p.z1did1.b0.cloudera.site:2181,cod-1e3xsc06g8mjm-leader0.exait3p.z1did1.b0.cloudera.site:2181@0x16c058f1" #166343 daemon prio=5 os_prio=0 tid=0x00007fc97caf2800 nid=0x648b waiting on condition [0x00007fc0e30f3000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007fc6fe058880> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.DelayQueue.poll(DelayQueue.java:259) at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:323) at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$$Lambda$164/1053864765.run(Unknown Source) at java.lang.Thread.run(Thread.java:750) "AsyncConn Chore Service.Chore.1" #166342 daemon prio=5 os_prio=0 tid=0x00007fc07f6f7800 nid=0x648a waiting on condition [0x00007fc0e33f6000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007fc6fe058a80> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1093) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:750) "ReadOnlyZKClient-cod-1e3xsc06g8mjm-master1.exait3p.z1did1.b0.cloudera.site:2181,cod-1e3xsc06g8mjm-master0.exait3p.z1did1.b0.cloudera.site:2181,cod-1e3xsc06g8mjm-leader0.exait3p.z1did1.b0.cloudera.site:2181@0x72b92415" #166331 daemon prio=5 os_prio=0 tid=0x00007fc0953e6800 nid=0x647a waiting on condition [0x00007fc0e7f41000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007fc6fe058cc0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.DelayQueue.poll(DelayQueue.java:259) at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:323) at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$$Lambda$164/1053864765.run(Unknown Source) at java.lang.Thread.run(Thread.java:750) "AsyncConn Chore Service.Chore.1" #166330 daemon prio=5 os_prio=0 tid=0x00007fc089340000 nid=0x6479 waiting on condition [0x00007fc0e4d0f000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007fc6fe058ec0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1093) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:750) "ReadOnlyZKClient-cod-1e3xsc06g8mjm-master1.exait3p.z1did1.b0.cloudera.site:2181,cod-1e3xsc06g8mjm-master0.exait3p.z1did1.b0.cloudera.site:2181,cod-1e3xsc06g8mjm-leader0.exait3p.z1did1.b0.cloudera.site:2181@0x1d095f5b" #166306 daemon prio=5 os_prio=0 tid=0x00007fc08467a000 nid=0x6456 waiting on condition [0x00007fc0e5d1f000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007fc6210c8378> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.DelayQueue.poll(DelayQueue.java:259) at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:323) at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$$Lambda$164/1053864765.run(Unknown Source) at java.lang.Thread.run(Thread.java:750) "AsyncConn Chore Service.Chore.1" #166305 daemon prio=5 os_prio=0 tid=0x00007fc0923a6800 nid=0x6455 waiting on condition [0x00007fc0d6ac7000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007fc6210c2378> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1093) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:750) "ReadOnlyZKClient-cod-1e3xsc06g8mjm-master1.exait3p.z1did1.b0.cloudera.site:2181,cod-1e3xsc06g8mjm-master0.exait3p.z1did1.b0.cloudera.site:2181,cod-1e3xsc06g8mjm-leader0.exait3p.z1did1.b0.cloudera.site:2181@0x74bc8b58" #166262 daemon prio=5 os_prio=0 tid=0x00007fc08d9f3800 nid=0x63fe waiting on condition [0x00007fc0c72f1000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007fc6210c8578> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.DelayQueue.poll(DelayQueue.java:259) at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:323) at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$$Lambda$164/1053864765.run(Unknown Source) at java.lang.Thread.run(Thread.java:750) "ReadOnlyZKClient-cod-1e3xsc06g8mjm-master1.exait3p.z1did1.b0.cloudera.site:2181,cod-1e3xsc06g8mjm-master0.exait3p.z1did1.b0.cloudera.site:2181,cod-1e3xsc06g8mjm-leader0.exait3p.z1did1.b0.cloudera.site:2181@0x36ab9d2f" #166261 daemon prio=5 os_prio=0 tid=0x0000558b1fbf2000 nid=0x63fd waiting on condition [0x00007fc0c73f2000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007fc6210c25b8> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.DelayQueue.poll(DelayQueue.java:259) at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:323) at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$$Lambda$164/1053864765.run(Unknown Source) at java.lang.Thread.run(Thread.java:750) "AsyncConn Chore Service.Chore.1" #166260 daemon prio=5 os_prio=0 tid=0x00007fc08c008800 nid=0x63fc waiting on condition [0x00007fc0c74f3000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007fc6210c8778> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1093) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:750) "AsyncConn Chore Service.Chore.1" #166259 daemon prio=5 os_prio=0 tid=0x0000558b254f8800 nid=0x63fb waiting on condition [0x00007fc0c75f4000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007fc6210c27b8> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1093) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:750) "ReadOnlyZKClient-cod-1e3xsc06g8mjm-master1.exait3p.z1did1.b0.cloudera.site:2181,cod-1e3xsc06g8mjm-master0.exait3p.z1did1.b0.cloudera.site:2181,cod-1e3xsc06g8mjm-leader0.exait3p.z1did1.b0.cloudera.site:2181@0x707e95b7" #166254 daemon prio=5 os_prio=0 tid=0x00007fc09fe48000 nid=0x63f6 waiting on condition [0x00007fc0c7af9000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007fc6210c4378> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.DelayQueue.poll(DelayQueue.java:259) at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:323) at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$$Lambda$164/1053864765.run(Unknown Source) at java.lang.Thread.run(Thread.java:750) "ReadOnlyZKClient-cod-1e3xsc06g8mjm-master1.exait3p.z1did1.b0.cloudera.site:2181,cod-1e3xsc06g8mjm-master0.exait3p.z1did1.b0.cloudera.site:2181,cod-1e3xsc06g8mjm-leader0.exait3p.z1did1.b0.cloudera.site:2181@0x25bee0d0" #166253 daemon prio=5 os_prio=0 tid=0x0000558b1d80e800 nid=0x63f5 waiting on condition [0x00007fc0c7bfa000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007fc6210c89b8> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.DelayQueue.poll(DelayQueue.java:259) at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:323) at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$$Lambda$164/1053864765.run(Unknown Source) at java.lang.Thread.run(Thread.java:750) "AsyncConn Chore Service.Chore.1" #166252 daemon prio=5 os_prio=0 tid=0x00007fc0930fc000 nid=0x63f4 waiting on condition [0x00007fc0c7cfb000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007fc6210c29f8> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1093) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:750) "AsyncConn Chore Service.Chore.1" #166251 daemon prio=5 os_prio=0 tid=0x0000558b269f3800 nid=0x63f3 waiting on condition [0x00007fc0c7dfc000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007fc6210c4578> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1093) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:750) "ReadOnlyZKClient-cod-1e3xsc06g8mjm-master1.exait3p.z1did1.b0.cloudera.site:2181,cod-1e3xsc06g8mjm-master0.exait3p.z1did1.b0.cloudera.site:2181,cod-1e3xsc06g8mjm-leader0.exait3p.z1did1.b0.cloudera.site:2181@0x13e63d1e" #166240 daemon prio=5 os_prio=0 tid=0x0000558b1ef12800 nid=0x63e4 waiting on condition [0x00007fc0ccb18000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007fc6210c2c38> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.DelayQueue.poll(DelayQueue.java:259) at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:323) at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$$Lambda$164/1053864765.run(Unknown Source) at java.lang.Thread.run(Thread.java:750) "AsyncConn Chore Service.Chore.1" #166239 daemon prio=5 os_prio=0 tid=0x0000558b202c3800 nid=0x63e3 waiting on condition [0x00007fc0ccc19000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007fc6210c2e38> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1093) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:750) "ReadOnlyZKClient-cod-1e3xsc06g8mjm-master1.exait3p.z1did1.b0.cloudera.site:2181,cod-1e3xsc06g8mjm-master0.exait3p.z1did1.b0.cloudera.site:2181,cod-1e3xsc06g8mjm-leader0.exait3p.z1did1.b0.cloudera.site:2181@0x63afa26a" #166235 daemon prio=5 os_prio=0 tid=0x00007fc086820800 nid=0x63df waiting on condition [0x00007fc0cd6d7000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007fc6210c47b8> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.DelayQueue.poll(DelayQueue.java:259) at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:323) at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$$Lambda$164/1053864765.run(Unknown Source) at java.lang.Thread.run(Thread.java:750) "AsyncConn Chore Service.Chore.1" #166234 daemon prio=5 os_prio=0 tid=0x00007fc08482a800 nid=0x63de waiting on condition [0x00007fc0dd8b9000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007fc6210c3078> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1093) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:750) "ReadOnlyZKClient-cod-1e3xsc06g8mjm-master1.exait3p.z1did1.b0.cloudera.site:2181,cod-1e3xsc06g8mjm-master0.exait3p.z1did1.b0.cloudera.site:2181,cod-1e3xsc06g8mjm-leader0.exait3p.z1did1.b0.cloudera.site:2181@0x68bdaf83" #166176 daemon prio=5 os_prio=0 tid=0x00007fc0a50f8800 nid=0x6378 waiting on condition [0x00007fc0ce2e3000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007fc6210c49b8> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.DelayQueue.poll(DelayQueue.java:259) at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:323) at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$$Lambda$164/1053864765.run(Unknown Source) at java.lang.Thread.run(Thread.java:750) "AsyncConn Chore Service.Chore.1" #166175 daemon prio=5 os_prio=0 tid=0x00007fc05d13e000 nid=0x6377 waiting on condition [0x00007fc0cfbfc000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007fc6210c32b8> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1093) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:750) "ReadOnlyZKClient-cod-1e3xsc06g8mjm-master1.exait3p.z1did1.b0.cloudera.site:2181,cod-1e3xsc06g8mjm-master0.exait3p.z1did1.b0.cloudera.site:2181,cod-1e3xsc06g8mjm-leader0.exait3p.z1did1.b0.cloudera.site:2181@0x3d6a48bb" #166145 daemon prio=5 os_prio=0 tid=0x00007fc05d401000 nid=0x633a waiting on condition [0x00007fc0cf0f1000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007fc6210c4bb8> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.DelayQueue.poll(DelayQueue.java:259) at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:323) at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$$Lambda$164/1053864765.run(Unknown Source) at java.lang.Thread.run(Thread.java:750) "AsyncConn Chore Service.Chore.1" #166144 daemon prio=5 os_prio=0 tid=0x00007fc11fb86000 nid=0x6339 waiting on condition [0x00007fc0cf1f2000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007fc6210cc378> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1093) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:750) "ReadOnlyZKClient-cod-1e3xsc06g8mjm-master1.exait3p.z1did1.b0.cloudera.site:2181,cod-1e3xsc06g8mjm-master0.exait3p.z1did1.b0.cloudera.site:2181,cod-1e3xsc06g8mjm-leader0.exait3p.z1did1.b0.cloudera.site:2181@0x13b0560a" #166141 daemon prio=5 os_prio=0 tid=0x00007fc08446a000 nid=0x6336 waiting on condition [0x00007fc0cf7f8000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007fc6210c34f8> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.DelayQueue.poll(DelayQueue.java:259) at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:323) at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$$Lambda$164/1053864765.run(Unknown Source) at java.lang.Thread.run(Thread.java:750) "AsyncConn Chore Service.Chore.1" #166140 daemon prio=5 os_prio=0 tid=0x00007fc091c28800 nid=0x6335 waiting on condition [0x00007fc0cf3f4000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007fc6210cc5b8> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1093) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:750) "ReadOnlyZKClient-cod-1e3xsc06g8mjm-master1.exait3p.z1did1.b0.cloudera.site:2181,cod-1e3xsc06g8mjm-master0.exait3p.z1did1.b0.cloudera.site:2181,cod-1e3xsc06g8mjm-leader0.exait3p.z1did1.b0.cloudera.site:2181@0x7d0bc58b" #166107 daemon prio=5 os_prio=0 tid=0x00007fc0a68f3800 nid=0x630c waiting on condition [0x00007fc0d71ce000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007fc6210c4db8> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.DelayQueue.poll(DelayQueue.java:259) at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:323) at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$$Lambda$164/1053864765.run(Unknown Source) at java.lang.Thread.run(Thread.java:750) "AsyncConn Chore Service.Chore.1" #166105 daemon prio=5 os_prio=0 tid=0x00007fc0a7c5b000 nid=0x6308 waiting on condition [0x00007fc0d74d1000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007fc6210c36f8> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1093) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:750) "ReadOnlyZKClient-cod-1e3xsc06g8mjm-master1.exait3p.z1did1.b0.cloudera.site:2181,cod-1e3xsc06g8mjm-master0.exait3p.z1did1.b0.cloudera.site:2181,cod-1e3xsc06g8mjm-leader0.exait3p.z1did1.b0.cloudera.site:2181@0x5de61058" #166103 daemon prio=5 os_prio=0 tid=0x00007fc0af331000 nid=0x6307 waiting on condition [0x00007fc0d75d2000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007fc6210cc7f8> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.DelayQueue.poll(DelayQueue.java:259) at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:323) at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$$Lambda$164/1053864765.run(Unknown Source) at java.lang.Thread.run(Thread.java:750) "AsyncConn Chore Service.Chore.1" #166102 daemon prio=5 os_prio=0 tid=0x00007fc09051c800 nid=0x6306 waiting on condition [0x00007fc0d76d3000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007fc6210c4fb8> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1093) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:750) "ReadOnlyZKClient-cod-1e3xsc06g8mjm-master1.exait3p.z1did1.b0.cloudera.site:2181,cod-1e3xsc06g8mjm-master0.exait3p.z1did1.b0.cloudera.site:2181,cod-1e3xsc06g8mjm-leader0.exait3p.z1did1.b0.cloudera.site:2181@0x07401b08" #166099 daemon prio=5 os_prio=0 tid=0x00007fc05d813000 nid=0x6303 waiting on condition [0x00007fc0ddabb000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007fc6210c3938> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.DelayQueue.poll(DelayQueue.java:259) at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:323) at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$$Lambda$164/1053864765.run(Unknown Source) at java.lang.Thread.run(Thread.java:750) "AsyncConn Chore Service.Chore.1" #166098 daemon prio=5 os_prio=0 tid=0x00007fc0b5a76000 nid=0x6302 waiting on condition [0x00007fc0dfec1000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007fc6210c51f8> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1093) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:750) "ReadOnlyZKClient-cod-1e3xsc06g8mjm-master1.exait3p.z1did1.b0.cloudera.site:2181,cod-1e3xsc06g8mjm-master0.exait3p.z1did1.b0.cloudera.site:2181,cod-1e3xsc06g8mjm-leader0.exait3p.z1did1.b0.cloudera.site:2181@0x6cab8507" #166092 daemon prio=5 os_prio=0 tid=0x00007fc0ae93b800 nid=0x62fb waiting on condition [0x00007fc0e11d4000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007fc6210c3b38> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.DelayQueue.poll(DelayQueue.java:259) at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:323) at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$$Lambda$164/1053864765.run(Unknown Source) at java.lang.Thread.run(Thread.java:750) "AsyncConn Chore Service.Chore.1" #166091 daemon prio=5 os_prio=0 tid=0x00007fc09e78b000 nid=0x62fa waiting on condition [0x00007fc0e13d6000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007fc6210c5438> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1093) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:750) "ReadOnlyZKClient-cod-1e3xsc06g8mjm-master1.exait3p.z1did1.b0.cloudera.site:2181,cod-1e3xsc06g8mjm-master0.exait3p.z1did1.b0.cloudera.site:2181,cod-1e3xsc06g8mjm-leader0.exait3p.z1did1.b0.cloudera.site:2181@0x3e086671" #166076 daemon prio=5 os_prio=0 tid=0x00007fc0b505c000 nid=0x62e6 waiting on condition [0x00007fc0ef7b9000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007fc6210ca378> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.DelayQueue.poll(DelayQueue.java:259) at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:323) at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$$Lambda$164/1053864765.run(Unknown Source) at java.lang.Thread.run(Thread.java:750) "ReadOnlyZKClient-cod-1e3xsc06g8mjm-master1.exait3p.z1did1.b0.cloudera.site:2181,cod-1e3xsc06g8mjm-master0.exait3p.z1did1.b0.cloudera.site:2181,cod-1e3xsc06g8mjm-leader0.exait3p.z1did1.b0.cloudera.site:2181@0x440d43a4" #166075 daemon prio=5 os_prio=0 tid=0x00007fc0a8010800 nid=0x62e5 waiting on condition [0x00007fc0e5012000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007fc6210c8bb8> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.DelayQueue.poll(DelayQueue.java:259) at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:323) at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$$Lambda$164/1053864765.run(Unknown Source) at java.lang.Thread.run(Thread.java:750) "AsyncConn Chore Service.Chore.1" #166074 daemon prio=5 os_prio=0 tid=0x00007fc0a7664800 nid=0x62e4 waiting on condition [0x00007fc0e0bce000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007fc6210ca578> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1093) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:750) "AsyncConn Chore Service.Chore.1" #166073 daemon prio=5 os_prio=0 tid=0x00007fc0af31f000 nid=0x62e3 waiting on condition [0x00007fc0e3f02000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007fc6210c8db8> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1093) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:750) "ReadOnlyZKClient-cod-1e3xsc06g8mjm-master1.exait3p.z1did1.b0.cloudera.site:2181,cod-1e3xsc06g8mjm-master0.exait3p.z1did1.b0.cloudera.site:2181,cod-1e3xsc06g8mjm-leader0.exait3p.z1did1.b0.cloudera.site:2181@0x574d7eb2" #166062 daemon prio=5 os_prio=0 tid=0x00007fc0b2b4c000 nid=0x62d5 waiting on condition [0x00007fc0e4b0d000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007fc6210c5678> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.DelayQueue.poll(DelayQueue.java:259) at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:323) at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$$Lambda$164/1053864765.run(Unknown Source) at java.lang.Thread.run(Thread.java:750) "AsyncConn Chore Service.Chore.1" #166061 daemon prio=5 os_prio=0 tid=0x00007fc08a3cb800 nid=0x62d4 waiting on condition [0x00007fc0e4c0e000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007fc6210ca7b8> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1093) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:750) "ReadOnlyZKClient-cod-1e3xsc06g8mjm-master1.exait3p.z1did1.b0.cloudera.site:2181,cod-1e3xsc06g8mjm-master0.exait3p.z1did1.b0.cloudera.site:2181,cod-1e3xsc06g8mjm-leader0.exait3p.z1did1.b0.cloudera.site:2181@0x036697ad" #166047 daemon prio=5 os_prio=0 tid=0x00007fc0b3422000 nid=0x62c4 waiting on condition [0x00007fc0e8648000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007fc6210c8ff8> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.DelayQueue.poll(DelayQueue.java:259) at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:323) at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$$Lambda$164/1053864765.run(Unknown Source) at java.lang.Thread.run(Thread.java:750) "AsyncConn Chore Service.Chore.1" #166046 daemon prio=5 os_prio=0 tid=0x00007fc096610000 nid=0x62c3 waiting on condition [0x00007fc0e894b000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007fc6210c5878> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1093) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:750) "ReadOnlyZKClient-cod-1e3xsc06g8mjm-master1.exait3p.z1did1.b0.cloudera.site:2181,cod-1e3xsc06g8mjm-master0.exait3p.z1did1.b0.cloudera.site:2181,cod-1e3xsc06g8mjm-leader0.exait3p.z1did1.b0.cloudera.site:2181@0x13b0db4a" #166033 daemon prio=5 os_prio=0 tid=0x00007fc05c584800 nid=0x62b1 waiting on condition [0x00007fc10e17e000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007fc6210c91f8> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.DelayQueue.poll(DelayQueue.java:259) at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:323) at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$$Lambda$164/1053864765.run(Unknown Source) at java.lang.Thread.run(Thread.java:750) "AsyncConn Chore Service.Chore.1" #166032 daemon prio=5 os_prio=0 tid=0x00007fc0b46f6800 nid=0x62b0 waiting on condition [0x00007fc0f3cfc000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007fc6210c93f8> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1093) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:750) "ReadOnlyZKClient-cod-1e3xsc06g8mjm-master1.exait3p.z1did1.b0.cloudera.site:2181,cod-1e3xsc06g8mjm-master0.exait3p.z1did1.b0.cloudera.site:2181,cod-1e3xsc06g8mjm-leader0.exait3p.z1did1.b0.cloudera.site:2181@0x3d100e1e" #166029 daemon prio=5 os_prio=0 tid=0x00007fc0d34da800 nid=0x62ad waiting on condition [0x00007fc0ddcbd000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007fc6210c9638> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.DelayQueue.poll(DelayQueue.java:259) at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:323) at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$$Lambda$164/1053864765.run(Unknown Source) at java.lang.Thread.run(Thread.java:750) "AsyncConn Chore Service.Chore.1" #166028 daemon prio=5 os_prio=0 tid=0x00007fc093418800 nid=0x62ac waiting on condition [0x00007fc0e9254000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007fc6210c3d38> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1093) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:750) "ReadOnlyZKClient-cod-1e3xsc06g8mjm-master1.exait3p.z1did1.b0.cloudera.site:2181,cod-1e3xsc06g8mjm-master0.exait3p.z1did1.b0.cloudera.site:2181,cod-1e3xsc06g8mjm-leader0.exait3p.z1did1.b0.cloudera.site:2181@0x6649ba2d" #166017 daemon prio=5 os_prio=0 tid=0x0000558b1dcee800 nid=0x6294 waiting on condition [0x00007fc0e7133000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007fc6210c9838> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.DelayQueue.poll(DelayQueue.java:259) at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:323) at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$$Lambda$164/1053864765.run(Unknown Source) at java.lang.Thread.run(Thread.java:750) "AsyncConn Chore Service.Chore.1" #166016 daemon prio=5 os_prio=0 tid=0x0000558b26a16800 nid=0x6293 waiting on condition [0x00007fc0f5312000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007fc6210c9a38> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1093) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:750) "ReadOnlyZKClient-cod-1e3xsc06g8mjm-master1.exait3p.z1did1.b0.cloudera.site:2181,cod-1e3xsc06g8mjm-master0.exait3p.z1did1.b0.cloudera.site:2181,cod-1e3xsc06g8mjm-leader0.exait3p.z1did1.b0.cloudera.site:2181@0x6f0a76bc" #165997 daemon prio=5 os_prio=0 tid=0x0000558b1ef17000 nid=0x623f waiting on condition [0x00007fc0ddbbc000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007fc6210c3f78> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.DelayQueue.poll(DelayQueue.java:259) at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:323) at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$$Lambda$164/1053864765.run(Unknown Source) at java.lang.Thread.run(Thread.java:750) "AsyncConn Chore Service.Chore.1" #165996 daemon prio=5 os_prio=0 tid=0x0000558b1dcf1800 nid=0x623e waiting on condition [0x00007fc10d05c000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007fc6210c9c78> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1093) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:750) "ReadOnlyZKClient-cod-1e3xsc06g8mjm-master1.exait3p.z1did1.b0.cloudera.site:2181,cod-1e3xsc06g8mjm-master0.exait3p.z1did1.b0.cloudera.site:2181,cod-1e3xsc06g8mjm-leader0.exait3p.z1did1.b0.cloudera.site:2181@0x6fce40ae" #165970 daemon prio=5 os_prio=0 tid=0x00007fc08d8cb000 nid=0x61f9 waiting on condition [0x00007fc0dd4b5000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007fc6210c9eb8> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.DelayQueue.poll(DelayQueue.java:259) at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:323) at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$$Lambda$164/1053864765.run(Unknown Source) at java.lang.Thread.run(Thread.java:750) "AsyncConn Chore Service.Chore.1" #165969 daemon prio=5 os_prio=0 tid=0x00007fc08ad40800 nid=0x61f8 waiting on condition [0x00007fc0dd7b8000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007fc6210c4178> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1093) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:750) "ReadOnlyZKClient-cod-1e3xsc06g8mjm-master1.exait3p.z1did1.b0.cloudera.site:2181,cod-1e3xsc06g8mjm-master0.exait3p.z1did1.b0.cloudera.site:2181,cod-1e3xsc06g8mjm-leader0.exait3p.z1did1.b0.cloudera.site:2181@0x523d9975" #162218 daemon prio=5 os_prio=0 tid=0x0000558b274e6800 nid=0x293c waiting on condition [0x00007fc0e10d3000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007fc562a3e9b8> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.DelayQueue.poll(DelayQueue.java:259) at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:323) at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$$Lambda$164/1053864765.run(Unknown Source) at java.lang.Thread.run(Thread.java:750) "ReadOnlyZKClient-cod-1e3xsc06g8mjm-master1.exait3p.z1did1.b0.cloudera.site:2181,cod-1e3xsc06g8mjm-master0.exait3p.z1did1.b0.cloudera.site:2181,cod-1e3xsc06g8mjm-leader0.exait3p.z1did1.b0.cloudera.site:2181@0x7a9f83a8" #162215 daemon prio=5 os_prio=0 tid=0x00007fc093474800 nid=0x2939 waiting on condition [0x00007fc0e682a000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007fc562a44160> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.DelayQueue.poll(DelayQueue.java:259) at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:323) at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$$Lambda$164/1053864765.run(Unknown Source) at java.lang.Thread.run(Thread.java:750) "AsyncConn Chore Service.Chore.1" #162214 daemon prio=5 os_prio=0 tid=0x00007fc069e52800 nid=0x2938 waiting on condition [0x00007fc0dd9ba000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007fc562a47d38> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1093) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:750) "AsyncConn Chore Service.Chore.1" #162213 daemon prio=5 os_prio=0 tid=0x0000558b266a8000 nid=0x2937 waiting on condition [0x00007fc0e3d00000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007fc562a44380> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1093) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:750) "ReadOnlyZKClient-cod-1e3xsc06g8mjm-master1.exait3p.z1did1.b0.cloudera.site:2181,cod-1e3xsc06g8mjm-master0.exait3p.z1did1.b0.cloudera.site:2181,cod-1e3xsc06g8mjm-leader0.exait3p.z1did1.b0.cloudera.site:2181@0x18f6c25e" #162136 daemon prio=5 os_prio=0 tid=0x00007fc06b0f0800 nid=0x2893 waiting on condition [0x00007fc0f5c1b000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007fc4ac2142a8> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.DelayQueue.poll(DelayQueue.java:259) at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:323) at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$$Lambda$164/1053864765.run(Unknown Source) at java.lang.Thread.run(Thread.java:750) "AsyncConn Chore Service.Chore.1" #162135 daemon prio=5 os_prio=0 tid=0x00007fc09140e800 nid=0x2892 waiting on condition [0x00007fc0e09cc000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007fc4ac20fb78> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1093) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:750) "ReadOnlyZKClient-cod-1e3xsc06g8mjm-master1.exait3p.z1did1.b0.cloudera.site:2181,cod-1e3xsc06g8mjm-master0.exait3p.z1did1.b0.cloudera.site:2181,cod-1e3xsc06g8mjm-leader0.exait3p.z1did1.b0.cloudera.site:2181@0x705d3f81" #162134 daemon prio=5 os_prio=0 tid=0x00007fc0b46f2800 nid=0x288f waiting on condition [0x00007fc0e8f51000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007fc4ac2144a8> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.DelayQueue.poll(DelayQueue.java:259) at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:323) at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$$Lambda$164/1053864765.run(Unknown Source) at java.lang.Thread.run(Thread.java:750) "AsyncConn Chore Service.Chore.1" #162133 daemon prio=5 os_prio=0 tid=0x00007fc11fadb000 nid=0x288e waiting on condition [0x00007fc0f25e5000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007fc4ac2146a8> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1093) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:750) "ReadOnlyZKClient-cod-1e3xsc06g8mjm-master1.exait3p.z1did1.b0.cloudera.site:2181,cod-1e3xsc06g8mjm-master0.exait3p.z1did1.b0.cloudera.site:2181,cod-1e3xsc06g8mjm-leader0.exait3p.z1did1.b0.cloudera.site:2181@0x3e31635b" #160449 daemon prio=5 os_prio=0 tid=0x00007fc0a5a24000 nid=0x12b6 waiting on condition [0x00007fc0e1fe2000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007fc50025acf8> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.DelayQueue.poll(DelayQueue.java:259) at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:323) at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$$Lambda$164/1053864765.run(Unknown Source) at java.lang.Thread.run(Thread.java:750) "AsyncConn Chore Service.Chore.1" #160448 daemon prio=5 os_prio=0 tid=0x00007fc0b4fa1800 nid=0x12b5 waiting on condition [0x00007fc0e24e7000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007fc50024b378> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1093) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:750) "ReadOnlyZKClient-cod-1e3xsc06g8mjm-master1.exait3p.z1did1.b0.cloudera.site:2181,cod-1e3xsc06g8mjm-master0.exait3p.z1did1.b0.cloudera.site:2181,cod-1e3xsc06g8mjm-leader0.exait3p.z1did1.b0.cloudera.site:2181@0x3dadfcbd" #160447 daemon prio=5 os_prio=0 tid=0x00007fc0b47fc000 nid=0x12b4 waiting on condition [0x00007fc0e2cef000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007fc50025af78> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.DelayQueue.poll(DelayQueue.java:259) at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:323) at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$$Lambda$164/1053864765.run(Unknown Source) at java.lang.Thread.run(Thread.java:750) "AsyncConn Chore Service.Chore.1" #160446 daemon prio=5 os_prio=0 tid=0x00007fc05f5f2800 nid=0x12b3 waiting on condition [0x00007fc0e36f9000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007fc500252b38> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1093) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:750) "ReadOnlyZKClient-cod-1e3xsc06g8mjm-master1.exait3p.z1did1.b0.cloudera.site:2181,cod-1e3xsc06g8mjm-master0.exait3p.z1did1.b0.cloudera.site:2181,cod-1e3xsc06g8mjm-leader0.exait3p.z1did1.b0.cloudera.site:2181@0x7f388019" #160335 daemon prio=5 os_prio=0 tid=0x00007fc11d077800 nid=0x11ba waiting on condition [0x00007fc0f5413000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007fc415a95688> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.DelayQueue.poll(DelayQueue.java:259) at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:323) at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$$Lambda$164/1053864765.run(Unknown Source) at java.lang.Thread.run(Thread.java:750) "AsyncConn Chore Service.Chore.1" #160334 daemon prio=5 os_prio=0 tid=0x00007fc11e513800 nid=0x11b9 waiting on condition [0x00007fc0e8143000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007fc50025b178> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1093) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:750) "ReadOnlyZKClient-cod-1e3xsc06g8mjm-master1.exait3p.z1did1.b0.cloudera.site:2181,cod-1e3xsc06g8mjm-master0.exait3p.z1did1.b0.cloudera.site:2181,cod-1e3xsc06g8mjm-leader0.exait3p.z1did1.b0.cloudera.site:2181@0x7aafab38" #160331 daemon prio=5 os_prio=0 tid=0x00007fc074a21000 nid=0x11b6 waiting on condition [0x00007fc109121000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007fc50024b5b8> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.DelayQueue.poll(DelayQueue.java:259) at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:323) at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$$Lambda$164/1053864765.run(Unknown Source) at java.lang.Thread.run(Thread.java:750) "AsyncConn Chore Service.Chore.1" #160330 daemon prio=5 os_prio=0 tid=0x00007fc09f791000 nid=0x11b5 waiting on condition [0x00007fc0e8c4e000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007fc415a95888> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1093) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:750) "ReadOnlyZKClient-cod-1e3xsc06g8mjm-master1.exait3p.z1did1.b0.cloudera.site:2181,cod-1e3xsc06g8mjm-master0.exait3p.z1did1.b0.cloudera.site:2181,cod-1e3xsc06g8mjm-leader0.exait3p.z1did1.b0.cloudera.site:2181@0x35ecfe66" #160083 daemon prio=5 os_prio=0 tid=0x00007fc074a1b800 nid=0xf06 waiting on condition [0x00007fc0cfafb000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007fc50025b3b8> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.DelayQueue.poll(DelayQueue.java:259) at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:323) at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$$Lambda$164/1053864765.run(Unknown Source) at java.lang.Thread.run(Thread.java:750) "ReadOnlyZKClient-cod-1e3xsc06g8mjm-master1.exait3p.z1did1.b0.cloudera.site:2181,cod-1e3xsc06g8mjm-master0.exait3p.z1did1.b0.cloudera.site:2181,cod-1e3xsc06g8mjm-leader0.exait3p.z1did1.b0.cloudera.site:2181@0x3a24601f" #160082 daemon prio=5 os_prio=0 tid=0x00007fc09f99a800 nid=0xf05 waiting on condition [0x00007fc0ea163000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007fc50024b7b8> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.DelayQueue.poll(DelayQueue.java:259) at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:323) at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$$Lambda$164/1053864765.run(Unknown Source) at java.lang.Thread.run(Thread.java:750) "AsyncConn Chore Service.Chore.1" #160081 daemon prio=5 os_prio=0 tid=0x00007fc084c20000 nid=0xf04 waiting on condition [0x00007fc0f6c2b000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007fc50025b5b8> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1093) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:750) "AsyncConn Chore Service.Chore.1" #160080 daemon prio=5 os_prio=0 tid=0x00007fc092071000 nid=0xf03 waiting on condition [0x00007fc0e5b1d000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007fc500252d78> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1093) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:750) "ReadOnlyZKClient-cod-1e3xsc06g8mjm-master1.exait3p.z1did1.b0.cloudera.site:2181,cod-1e3xsc06g8mjm-master0.exait3p.z1did1.b0.cloudera.site:2181,cod-1e3xsc06g8mjm-leader0.exait3p.z1did1.b0.cloudera.site:2181@0x6f56c717" #159958 daemon prio=5 os_prio=0 tid=0x00007fc08f9d7800 nid=0xdc2 waiting on condition [0x00007fc0e8e50000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007fc50025b7f8> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.DelayQueue.poll(DelayQueue.java:259) at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:323) at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$$Lambda$164/1053864765.run(Unknown Source) at java.lang.Thread.run(Thread.java:750) "AsyncConn Chore Service.Chore.1" #159957 daemon prio=5 os_prio=0 tid=0x00007fc08160e800 nid=0xdc1 waiting on condition [0x00007fc0e4a0c000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007fc50024b9b8> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1093) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:750) "ReadOnlyZKClient-cod-1e3xsc06g8mjm-master1.exait3p.z1did1.b0.cloudera.site:2181,cod-1e3xsc06g8mjm-master0.exait3p.z1did1.b0.cloudera.site:2181,cod-1e3xsc06g8mjm-leader0.exait3p.z1did1.b0.cloudera.site:2181@0x02ef184d" #159954 daemon prio=5 os_prio=0 tid=0x00007fc0b591a800 nid=0xdbe waiting on condition [0x00007fc0f1cdc000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007fc500260ff8> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.DelayQueue.poll(DelayQueue.java:259) at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:323) at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$$Lambda$164/1053864765.run(Unknown Source) at java.lang.Thread.run(Thread.java:750) "AsyncConn Chore Service.Chore.1" #159953 daemon prio=5 os_prio=0 tid=0x00007fc0b5bae000 nid=0xdbd waiting on condition [0x00007fc0f5110000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007fc500256578> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1093) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:750) "ReadOnlyZKClient-cod-1e3xsc06g8mjm-master1.exait3p.z1did1.b0.cloudera.site:2181,cod-1e3xsc06g8mjm-master0.exait3p.z1did1.b0.cloudera.site:2181,cod-1e3xsc06g8mjm-leader0.exait3p.z1did1.b0.cloudera.site:2181@0x12dda7b0" #159899 daemon prio=5 os_prio=0 tid=0x00007fc088f29800 nid=0xd63 waiting on condition [0x00007fc0e5a1c000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007fc2b7d62178> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.DelayQueue.poll(DelayQueue.java:259) at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:323) at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$$Lambda$164/1053864765.run(Unknown Source) at java.lang.Thread.run(Thread.java:750) "AsyncConn Chore Service.Chore.1" #159898 daemon prio=5 os_prio=0 tid=0x00007fc0806bb000 nid=0xd60 waiting on condition [0x00007fc0f4b0a000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007fc2b7d66440> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1093) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:750) "ReadOnlyZKClient-cod-1e3xsc06g8mjm-master1.exait3p.z1did1.b0.cloudera.site:2181,cod-1e3xsc06g8mjm-master0.exait3p.z1did1.b0.cloudera.site:2181,cod-1e3xsc06g8mjm-leader0.exait3p.z1did1.b0.cloudera.site:2181@0x11e79238" #159895 daemon prio=5 os_prio=0 tid=0x00007fc0b5dce000 nid=0xd5f waiting on condition [0x00007fc0ed092000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007fc2b7d64178> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.DelayQueue.poll(DelayQueue.java:259) at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:323) at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$$Lambda$164/1053864765.run(Unknown Source) at java.lang.Thread.run(Thread.java:750) "AsyncConn Chore Service.Chore.1" #159894 daemon prio=5 os_prio=0 tid=0x00007fc11cf6e000 nid=0xd5e waiting on condition [0x00007fc0eab6d000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007fc2b7d66680> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1093) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:750) "ReadOnlyZKClient-cod-1e3xsc06g8mjm-master1.exait3p.z1did1.b0.cloudera.site:2181,cod-1e3xsc06g8mjm-master0.exait3p.z1did1.b0.cloudera.site:2181,cod-1e3xsc06g8mjm-leader0.exait3p.z1did1.b0.cloudera.site:2181@0x4dfaac2c" #159731 daemon prio=5 os_prio=0 tid=0x00007fc084e13000 nid=0xbb8 waiting on condition [0x00007fc0eb476000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007fc50025b9f8> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.DelayQueue.poll(DelayQueue.java:259) at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:323) at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$$Lambda$164/1053864765.run(Unknown Source) at java.lang.Thread.run(Thread.java:750) "AsyncConn Chore Service.Chore.1" #159730 daemon prio=5 os_prio=0 tid=0x00007fc0aeeb3000 nid=0xbb7 waiting on condition [0x00007fc0e5c1e000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007fc5002611f8> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1093) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:750) "ReadOnlyZKClient-cod-1e3xsc06g8mjm-master1.exait3p.z1did1.b0.cloudera.site:2181,cod-1e3xsc06g8mjm-master0.exait3p.z1did1.b0.cloudera.site:2181,cod-1e3xsc06g8mjm-leader0.exait3p.z1did1.b0.cloudera.site:2181@0x530b7c41" #159727 daemon prio=5 os_prio=0 tid=0x0000558b27905800 nid=0xbb4 waiting on condition [0x00007fc0ea466000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007fc50024bbf8> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.DelayQueue.poll(DelayQueue.java:259) at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:323) at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$$Lambda$164/1053864765.run(Unknown Source) at java.lang.Thread.run(Thread.java:750) "AsyncConn Chore Service.Chore.1" #159726 daemon prio=5 os_prio=0 tid=0x0000558b17396000 nid=0xbb3 waiting on condition [0x00007fc0fb875000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007fc500261438> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1093) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:750) "RS_REFRESH_PEER-regionserver/cod-1e3xsc06g8mjm-worker124:16020-1.replicationSource,kafkaproxy.replicationSource.shippercod-1e3xsc06g8mjm-worker124.exait3p.z1did1.b0.cloudera.site%2C16020%2C1668705752838.cod-1e3xsc06g8mjm-worker124.exait3p.z1did1.b0.cloudera.site%2C16020%2C1668705752838.regiongroup-0,kafkaproxy" #159036 daemon prio=5 os_prio=0 tid=0x00007fc09133f800 nid=0x2f4 waiting on condition [0x00007fc0ea567000] java.lang.Thread.State: WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007fc912e8a4e8> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) at java.util.concurrent.ExecutorCompletionService.take(ExecutorCompletionService.java:193) at org.apache.hadoop.hbase.replication.regionserver.HBaseInterClusterReplicationEndpoint.parallelReplicate(HBaseInterClusterReplicationEndpoint.java:480) at org.apache.hadoop.hbase.replication.regionserver.HBaseInterClusterReplicationEndpoint.replicate(HBaseInterClusterReplicationEndpoint.java:542) at org.apache.hadoop.hbase.replication.regionserver.ReplicationSourceShipper.shipEdits(ReplicationSourceShipper.java:189) at org.apache.hadoop.hbase.replication.regionserver.ReplicationSourceShipper.run(ReplicationSourceShipper.java:117) "RS_REFRESH_PEER-regionserver/cod-1e3xsc06g8mjm-worker124:16020-1.replicationSource,kafkaproxy.replicationSource.wal-reader.cod-1e3xsc06g8mjm-worker124.exait3p.z1did1.b0.cloudera.site%2C16020%2C1668705752838.cod-1e3xsc06g8mjm-worker124.exait3p.z1did1.b0.cloudera.site%2C16020%2C1668705752838.regiongroup-0,kafkaproxy" #159037 daemon prio=5 os_prio=0 tid=0x00007fc090f15000 nid=0x2f3 waiting on condition [0x00007fc0ea668000] java.lang.Thread.State: TIMED_WAITING (sleeping) at java.lang.Thread.sleep(Native Method) at org.apache.hadoop.hbase.replication.regionserver.ReplicationSourceWALReader.handleEmptyWALEntryBatch(ReplicationSourceWALReader.java:236) at org.apache.hadoop.hbase.replication.regionserver.ReplicationSourceWALReader.run(ReplicationSourceWALReader.java:144) "zk-event-processor-pool-0" #159035 daemon prio=5 os_prio=0 tid=0x00007fc08d239800 nid=0x2f1 waiting on condition [0x00007fc0ea769000] java.lang.Thread.State: WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007fc415a95ed0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:750) "Thread-22909" #159034 daemon prio=5 os_prio=0 tid=0x0000558b1c888800 nid=0x2f0 waiting on condition [0x00007fc0f7a39000] java.lang.Thread.State: TIMED_WAITING (sleeping) at java.lang.Thread.sleep(Native Method) at org.apache.zookeeper.Login$1.run(Login.java:199) at java.lang.Thread.run(Thread.java:750) "RS_REFRESH_PEER-regionserver/cod-1e3xsc06g8mjm-worker124:16020-1.replicationSource,kafkaproxy-EventThread" #159033 daemon prio=5 os_prio=0 tid=0x00007fc0858d7000 nid=0x2ef waiting on condition [0x00007fc0ea86a000] java.lang.Thread.State: WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007fc415a96370> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) at org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:503) "RS_REFRESH_PEER-regionserver/cod-1e3xsc06g8mjm-worker124:16020-1.replicationSource,kafkaproxy-SendThread(cod-1e3xsc06g8mjm-master1.exait3p.z1did1.b0.cloudera.site:2181)" #159032 daemon prio=5 os_prio=0 tid=0x00007fc09da5d000 nid=0x2ee runnable [0x00007fc0ea96b000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007fc415a96628> (a sun.nio.ch.Util$3) - locked <0x00007fc415a96638> (a java.util.Collections$UnmodifiableSet) - locked <0x00007fc415a965e0> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.zookeeper.ClientCnxnSocketNIO.doTransport(ClientCnxnSocketNIO.java:345) at org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1225) "ReadOnlyZKClient-cod-1e3xsc06g8mjm-master0.exait3p.z1did1.b0.cloudera.site:2181,cod-1e3xsc06g8mjm-master1.exait3p.z1did1.b0.cloudera.site:2181,cod-1e3xsc06g8mjm-leader0.exait3p.z1did1.b0.cloudera.site:2181@0x663aa808" #159028 daemon prio=5 os_prio=0 tid=0x00007fc0aa30b000 nid=0x2ea waiting on condition [0x00007fc0edea0000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007fc415a968c0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.DelayQueue.poll(DelayQueue.java:259) at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:323) at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$$Lambda$164/1053864765.run(Unknown Source) at java.lang.Thread.run(Thread.java:750) "AsyncConn Chore Service.Chore.1" #159027 daemon prio=5 os_prio=0 tid=0x00007fc07710a000 nid=0x2e9 waiting on condition [0x00007fc0ef8ba000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007fc415a96ac0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1093) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:750) "ReadOnlyZKClient-cod-1e3xsc06g8mjm-master1.exait3p.z1did1.b0.cloudera.site:2181,cod-1e3xsc06g8mjm-master0.exait3p.z1did1.b0.cloudera.site:2181,cod-1e3xsc06g8mjm-leader0.exait3p.z1did1.b0.cloudera.site:2181@0x608f00a5" #158946 daemon prio=5 os_prio=0 tid=0x00007fc09c1e0800 nid=0x7f2c waiting on condition [0x00007fc0d69c6000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007fc415a96ee0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.DelayQueue.poll(DelayQueue.java:259) at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:323) at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$$Lambda$164/1053864765.run(Unknown Source) at java.lang.Thread.run(Thread.java:750) "AsyncConn Chore Service.Chore.1" #158945 daemon prio=5 os_prio=0 tid=0x00007fc0d3e2f800 nid=0x7f2b waiting on condition [0x00007fc10c940000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007fc415ab1ca8> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1093) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:750) "ReadOnlyZKClient-cod-1e3xsc06g8mjm-master1.exait3p.z1did1.b0.cloudera.site:2181,cod-1e3xsc06g8mjm-master0.exait3p.z1did1.b0.cloudera.site:2181,cod-1e3xsc06g8mjm-leader0.exait3p.z1did1.b0.cloudera.site:2181@0x3fd3ddc5" #158876 daemon prio=5 os_prio=0 tid=0x00007fc0560ec000 nid=0x7c29 waiting on condition [0x00007fc0e0fd2000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007fc415ab1ee8> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.DelayQueue.poll(DelayQueue.java:259) at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:323) at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$$Lambda$164/1053864765.run(Unknown Source) at java.lang.Thread.run(Thread.java:750) "AsyncConn Chore Service.Chore.1" #158875 daemon prio=5 os_prio=0 tid=0x00007fc0bc601800 nid=0x7c28 waiting on condition [0x00007fc0f22e2000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007fc415ab20e8> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1093) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:750) "ReadOnlyZKClient-cod-1e3xsc06g8mjm-master1.exait3p.z1did1.b0.cloudera.site:2181,cod-1e3xsc06g8mjm-master0.exait3p.z1did1.b0.cloudera.site:2181,cod-1e3xsc06g8mjm-leader0.exait3p.z1did1.b0.cloudera.site:2181@0x4fb90c1c" #158763 daemon prio=5 os_prio=0 tid=0x00007fc0a6f2c800 nid=0x75e2 waiting on condition [0x00007fc0ef4b6000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007fc54225dd78> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.DelayQueue.poll(DelayQueue.java:259) at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:323) at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$$Lambda$164/1053864765.run(Unknown Source) at java.lang.Thread.run(Thread.java:750) "AsyncConn Chore Service.Chore.1" #158762 daemon prio=5 os_prio=0 tid=0x00007fc05d4b3000 nid=0x75e1 waiting on condition [0x00007fc0f5e1d000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007fc54225fdb8> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1093) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:750) "ReadOnlyZKClient-cod-1e3xsc06g8mjm-master1.exait3p.z1did1.b0.cloudera.site:2181,cod-1e3xsc06g8mjm-master0.exait3p.z1did1.b0.cloudera.site:2181,cod-1e3xsc06g8mjm-leader0.exait3p.z1did1.b0.cloudera.site:2181@0x0ad843c4" #158712 daemon prio=5 os_prio=0 tid=0x00007fc0af92d000 nid=0x7385 waiting on condition [0x00007fc0efcbc000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007fc415ab2328> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.DelayQueue.poll(DelayQueue.java:259) at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:323) at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$$Lambda$164/1053864765.run(Unknown Source) at java.lang.Thread.run(Thread.java:750) "AsyncConn Chore Service.Chore.1" #158711 daemon prio=5 os_prio=0 tid=0x00007fc054861800 nid=0x7384 waiting on condition [0x00007fc0d47d1000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007fc415ab2528> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1093) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:750) "ReadOnlyZKClient-cod-1e3xsc06g8mjm-master1.exait3p.z1did1.b0.cloudera.site:2181,cod-1e3xsc06g8mjm-master0.exait3p.z1did1.b0.cloudera.site:2181,cod-1e3xsc06g8mjm-leader0.exait3p.z1did1.b0.cloudera.site:2181@0x381972bf" #158459 daemon prio=5 os_prio=0 tid=0x00007fc11d5f4000 nid=0x6244 waiting on condition [0x00007fc0f9d5a000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007fc415ab2768> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.DelayQueue.poll(DelayQueue.java:259) at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:323) at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$$Lambda$164/1053864765.run(Unknown Source) at java.lang.Thread.run(Thread.java:750) "ReadOnlyZKClient-cod-1e3xsc06g8mjm-master1.exait3p.z1did1.b0.cloudera.site:2181,cod-1e3xsc06g8mjm-master0.exait3p.z1did1.b0.cloudera.site:2181,cod-1e3xsc06g8mjm-leader0.exait3p.z1did1.b0.cloudera.site:2181@0x547c3c90" #158458 daemon prio=5 os_prio=0 tid=0x00007fc0b23be800 nid=0x6243 waiting on condition [0x00007fc0d43cd000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007fc415ab2968> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.DelayQueue.poll(DelayQueue.java:259) at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:323) at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$$Lambda$164/1053864765.run(Unknown Source) at java.lang.Thread.run(Thread.java:750) "AsyncConn Chore Service.Chore.1" #158457 daemon prio=5 os_prio=0 tid=0x00007fc0b4117000 nid=0x6242 waiting on condition [0x00007fc0f23e3000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007fc415ab2b68> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1093) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:750) "AsyncConn Chore Service.Chore.1" #158456 daemon prio=5 os_prio=0 tid=0x00007fc08c591800 nid=0x6241 waiting on condition [0x00007fc0f24e4000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007fc415ab2da8> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1093) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:750) "PHOENIX-SCANNER-RENEW-LEASE-thread-9" #117707 daemon prio=5 os_prio=0 tid=0x00007fc0a55d7000 nid=0x61c9 waiting on condition [0x00007fc0cd01d000] java.lang.Thread.State: WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007fc415ab2fe8> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1088) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:750) "PHOENIX-SCANNER-RENEW-LEASE-thread-10" #117706 daemon prio=5 os_prio=0 tid=0x0000558b29b9e000 nid=0x61c8 waiting on condition [0x00007fc0cc58e000] java.lang.Thread.State: WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007fc415ab2fe8> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1088) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:750) "PHOENIX-SCANNER-RENEW-LEASE-thread-8" #117705 daemon prio=5 os_prio=0 tid=0x00007fc084c2a000 nid=0x61c7 waiting on condition [0x00007fc0cc48d000] java.lang.Thread.State: WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007fc415ab2fe8> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1088) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:750) "PHOENIX-SCANNER-RENEW-LEASE-thread-7" #117704 daemon prio=5 os_prio=0 tid=0x0000558b26a64000 nid=0x61c6 waiting on condition [0x00007fc0cd21f000] java.lang.Thread.State: WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007fc415ab2fe8> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1088) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:750) "PHOENIX-SCANNER-RENEW-LEASE-thread-6" #117703 daemon prio=5 os_prio=0 tid=0x0000558b27436800 nid=0x61c5 waiting on condition [0x00007fc0cc790000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007fc415ab2fe8> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1093) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:750) "PHOENIX-SCANNER-RENEW-LEASE-thread-5" #117702 daemon prio=5 os_prio=0 tid=0x0000558b141a6000 nid=0x61c4 waiting on condition [0x00007fc0cc38c000] java.lang.Thread.State: WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007fc415ab2fe8> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1088) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:750) "PHOENIX-SCANNER-RENEW-LEASE-thread-4" #117701 daemon prio=5 os_prio=0 tid=0x0000558b29b9c000 nid=0x61c3 waiting on condition [0x00007fc0cd320000] java.lang.Thread.State: WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007fc415ab2fe8> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1088) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:750) "PHOENIX-SCANNER-RENEW-LEASE-thread-3" #117700 daemon prio=5 os_prio=0 tid=0x0000558b2a434000 nid=0x61c2 waiting on condition [0x00007fc0cd11e000] java.lang.Thread.State: WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007fc415ab2fe8> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1088) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:750) "PHOENIX-SCANNER-RENEW-LEASE-thread-2" #117699 daemon prio=5 os_prio=0 tid=0x0000558b21534000 nid=0x61c1 waiting on condition [0x00007fc0cd522000] java.lang.Thread.State: WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007fc415ab2fe8> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1088) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:750) "PHOENIX-SCANNER-RENEW-LEASE-thread-1" #117698 daemon prio=5 os_prio=0 tid=0x0000558b275a1800 nid=0x61c0 waiting on condition [0x00007fc0cd421000] java.lang.Thread.State: WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007fc415ab2fe8> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1088) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:750) "ReadOnlyZKClient-cod-1e3xsc06g8mjm-master1.exait3p.z1did1.b0.cloudera.site:2181,cod-1e3xsc06g8mjm-master0.exait3p.z1did1.b0.cloudera.site:2181,cod-1e3xsc06g8mjm-leader0.exait3p.z1did1.b0.cloudera.site:2181@0x594362fa" #117658 daemon prio=5 os_prio=0 tid=0x0000558b20934800 nid=0x613c waiting on condition [0x00007fc0eac6e000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007fc415ab61a0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.DelayQueue.poll(DelayQueue.java:259) at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:323) at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$$Lambda$164/1053864765.run(Unknown Source) at java.lang.Thread.run(Thread.java:750) "AsyncConn Chore Service.Chore.1" #117657 daemon prio=5 os_prio=0 tid=0x0000558b1fb97800 nid=0x613b waiting on condition [0x00007fc0e4709000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007fc415ab63a0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1093) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:750) "ShortCircuitCache_SlotReleaser" #81119 daemon prio=5 os_prio=0 tid=0x00007fc0c8347800 nid=0x7f83 waiting on condition [0x00007fc0fe09d000] java.lang.Thread.State: WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007fc1dbb34e80> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1081) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:750) "Hashed wheel timer #3" #80549 daemon prio=5 os_prio=0 tid=0x00007fc093ba6800 nid=0x7837 waiting on condition [0x00007fc0f8645000] java.lang.Thread.State: TIMED_WAITING (sleeping) at java.lang.Thread.sleep(Native Method) at com.cloudera.cmf.event.shaded.org.jboss.netty.util.HashedWheelTimer$Worker.waitForNextTick(HashedWheelTimer.java:445) at com.cloudera.cmf.event.shaded.org.jboss.netty.util.HashedWheelTimer$Worker.run(HashedWheelTimer.java:364) at com.cloudera.cmf.event.shaded.org.jboss.netty.util.ThreadRenamingRunnable.run(ThreadRenamingRunnable.java:108) at java.lang.Thread.run(Thread.java:750) "New I/O worker #6" #80551 daemon prio=5 os_prio=0 tid=0x0000558b1e715800 nid=0x7836 runnable [0x00007fc10e27f000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007fc536bcf210> (a sun.nio.ch.Util$3) - locked <0x00007fc536bcf220> (a java.util.Collections$UnmodifiableSet) - locked <0x00007fc536bcf1c8> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at com.cloudera.cmf.event.shaded.org.jboss.netty.channel.socket.nio.SelectorUtil.select(SelectorUtil.java:68) at com.cloudera.cmf.event.shaded.org.jboss.netty.channel.socket.nio.AbstractNioSelector.select(AbstractNioSelector.java:434) at com.cloudera.cmf.event.shaded.org.jboss.netty.channel.socket.nio.AbstractNioSelector.run(AbstractNioSelector.java:212) at com.cloudera.cmf.event.shaded.org.jboss.netty.channel.socket.nio.AbstractNioWorker.run(AbstractNioWorker.java:89) at com.cloudera.cmf.event.shaded.org.jboss.netty.channel.socket.nio.NioWorker.run(NioWorker.java:178) at com.cloudera.cmf.event.shaded.org.jboss.netty.util.ThreadRenamingRunnable.run(ThreadRenamingRunnable.java:108) at com.cloudera.cmf.event.shaded.org.jboss.netty.util.internal.DeadLockProofWorker$1.run(DeadLockProofWorker.java:42) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:750) "New I/O boss #5" #80550 daemon prio=5 os_prio=0 tid=0x0000558b140b0000 nid=0x7835 runnable [0x00007fc0dddbe000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007fc536bbf808> (a sun.nio.ch.Util$3) - locked <0x00007fc536bbf818> (a java.util.Collections$UnmodifiableSet) - locked <0x00007fc536bbf7c0> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at com.cloudera.cmf.event.shaded.org.jboss.netty.channel.socket.nio.SelectorUtil.select(SelectorUtil.java:68) at com.cloudera.cmf.event.shaded.org.jboss.netty.channel.socket.nio.AbstractNioSelector.select(AbstractNioSelector.java:434) at com.cloudera.cmf.event.shaded.org.jboss.netty.channel.socket.nio.AbstractNioSelector.run(AbstractNioSelector.java:212) at com.cloudera.cmf.event.shaded.org.jboss.netty.channel.socket.nio.NioClientBoss.run(NioClientBoss.java:42) at com.cloudera.cmf.event.shaded.org.jboss.netty.util.ThreadRenamingRunnable.run(ThreadRenamingRunnable.java:108) at com.cloudera.cmf.event.shaded.org.jboss.netty.util.internal.DeadLockProofWorker$1.run(DeadLockProofWorker.java:42) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:750) "ShortCircuitCache_Cleaner" #80300 daemon prio=5 os_prio=0 tid=0x00007fc08bdbe800 nid=0x760e waiting on condition [0x00007fc0c2314000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007fc1dbb34c30> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1093) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:750) "member: 'cod-1e3xsc06g8mjm-worker124.exait3p.z1did1.b0.cloudera.site,16020,1668705752838' subprocedure-pool-0" #80156 daemon prio=5 os_prio=0 tid=0x00007fc06a10a800 nid=0x7386 waiting on condition [0x00007fc0c4233000] java.lang.Thread.State: WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007fc1dc267998> (a java.util.concurrent.SynchronousQueue$TransferStack) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:458) at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362) at java.util.concurrent.SynchronousQueue.take(SynchronousQueue.java:924) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:750) "Session-Scheduler-3d4d4187-1" #78302 prio=5 os_prio=0 tid=0x00007fc0a502c800 nid=0x5ffa waiting on condition [0x00007fc0ec385000] java.lang.Thread.State: WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007fc1dbd46a10> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1081) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:750) "regionserver/cod-1e3xsc06g8mjm-worker124:16020.Chore.3" #78126 daemon prio=5 os_prio=0 tid=0x00007fc05d4f0800 nid=0x5922 waiting on condition [0x00007fc1023de000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007fc1c9fe5e08> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1093) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:750) "RS_REFRESH_PEER-regionserver/cod-1e3xsc06g8mjm-worker124:16020-1" #76760 daemon prio=5 os_prio=0 tid=0x00007fc089445000 nid=0x3a67 waiting on condition [0x00007fc0e8a4c000] java.lang.Thread.State: WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007fc1dc07c018> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:750) "zkConnectionManagerCallback-5-thread-1" #76522 prio=5 os_prio=0 tid=0x0000558b14c32000 nid=0x3653 waiting on condition [0x00007fc0f0aca000] java.lang.Thread.State: WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007fc50023f378> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:750) "Thread-4311" #76521 daemon prio=5 os_prio=0 tid=0x00007fc11f9c5000 nid=0x3652 waiting on condition [0x00007fc0e5113000] java.lang.Thread.State: TIMED_WAITING (sleeping) at java.lang.Thread.sleep(Native Method) at org.apache.zookeeper.Login$1.run(Login.java:199) at java.lang.Thread.run(Thread.java:750) "org.apache.ranger.audit.queue.AuditBatchQueue1-EventThread" #76520 daemon prio=5 os_prio=0 tid=0x0000558b26be6800 nid=0x3650 waiting on condition [0x00007fc0f3fff000] java.lang.Thread.State: WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007fc50023f818> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) at org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:503) "org.apache.ranger.audit.queue.AuditBatchQueue1-SendThread(exait3p-glin-dl-gateway1.exait3p.z1did1.b0.cloudera.site:2181)" #76519 daemon prio=5 os_prio=0 tid=0x0000558b208cc800 nid=0x364f runnable [0x00007fc0e8547000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007fc50023fad0> (a sun.nio.ch.Util$3) - locked <0x00007fc50023fae0> (a java.util.Collections$UnmodifiableSet) - locked <0x00007fc50023fa88> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.zookeeper.ClientCnxnSocketNIO.doTransport(ClientCnxnSocketNIO.java:345) at org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1225) "ReadOnlyZKClient-cod-1e3xsc06g8mjm-master1.exait3p.z1did1.b0.cloudera.site:2181,cod-1e3xsc06g8mjm-master0.exait3p.z1did1.b0.cloudera.site:2181,cod-1e3xsc06g8mjm-leader0.exait3p.z1did1.b0.cloudera.site:2181@0x73b80e92" #31413 daemon prio=5 os_prio=0 tid=0x0000558b1e2fd000 nid=0x1316 waiting on condition [0x00007fc1018d3000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007fc401667640> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.DelayQueue.poll(DelayQueue.java:259) at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:323) at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$$Lambda$164/1053864765.run(Unknown Source) at java.lang.Thread.run(Thread.java:750) "AsyncConn Chore Service.Chore.1" #31412 daemon prio=5 os_prio=0 tid=0x0000558b1db50800 nid=0x1315 waiting on condition [0x00007fc0e7d3f000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007fc40166c978> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1093) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:750) "ReadOnlyZKClient-cod-1e3xsc06g8mjm-master1.exait3p.z1did1.b0.cloudera.site:2181,cod-1e3xsc06g8mjm-master0.exait3p.z1did1.b0.cloudera.site:2181,cod-1e3xsc06g8mjm-leader0.exait3p.z1did1.b0.cloudera.site:2181@0x1eb82c1e" #31411 daemon prio=5 os_prio=0 tid=0x00007fc09457c000 nid=0x1314 waiting on condition [0x00007fc0e9557000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007fc401667860> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.DelayQueue.poll(DelayQueue.java:259) at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:323) at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$$Lambda$164/1053864765.run(Unknown Source) at java.lang.Thread.run(Thread.java:750) "AsyncConn Chore Service.Chore.1" #31410 daemon prio=5 os_prio=0 tid=0x00007fc97c807800 nid=0x1313 waiting on condition [0x00007fc0efdbd000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007fc401667a80> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1093) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:750) "regionserver/cod-1e3xsc06g8mjm-worker124:16020-splits-0" #31399 daemon prio=5 os_prio=0 tid=0x00007fc0a8ba6000 nid=0x12f8 waiting on condition [0x00007fc0e9658000] java.lang.Thread.State: WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007fc1dc839e20> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:750) "regionserver/cod-1e3xsc06g8mjm-worker124:16020.Chore.2" #26585 daemon prio=5 os_prio=0 tid=0x00007fc068883000 nid=0x10ce waiting on condition [0x00007fc0f2ef0000] java.lang.Thread.State: WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007fc1c9fe5e08> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1088) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:750) "RS_CLOSE_REGION-regionserver/cod-1e3xsc06g8mjm-worker124:16020-2" #26321 daemon prio=5 os_prio=0 tid=0x00007fc06485c800 nid=0xa68 waiting on condition [0x00007fc0e995b000] java.lang.Thread.State: WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007fc1dc361958> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:750) "RS_CLOSE_REGION-regionserver/cod-1e3xsc06g8mjm-worker124:16020-1" #26153 daemon prio=5 os_prio=0 tid=0x0000558b22480800 nid=0x634 waiting on condition [0x00007fc1002bd000] java.lang.Thread.State: WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007fc1dc361958> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:750) "RS_CLOSE_REGION-regionserver/cod-1e3xsc06g8mjm-worker124:16020-0" #26049 daemon prio=5 os_prio=0 tid=0x00007fc0ae95e800 nid=0x289 waiting on condition [0x00007fc0e12d5000] java.lang.Thread.State: WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007fc1dc361958> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:750) "phoenix-update-statistics-3" #23962 daemon prio=5 os_prio=0 tid=0x0000558b1de51800 nid=0x4ad9 waiting on condition [0x00007fc0b84a4000] java.lang.Thread.State: WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007fc518c24578> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:750) "phoenix-update-statistics-2" #23942 daemon prio=5 os_prio=0 tid=0x00007fc083399000 nid=0x4abf waiting on condition [0x00007fc0e6729000] java.lang.Thread.State: WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007fc518c24578> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:750) "phoenix-update-statistics-1" #18959 daemon prio=5 os_prio=0 tid=0x00007fc07ffe7800 nid=0x643 waiting on condition [0x00007fc0c4334000] java.lang.Thread.State: WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007fc518c24578> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:750) "phoenix-update-statistics-0" #18806 daemon prio=5 os_prio=0 tid=0x00007fc09d824800 nid=0x500 waiting on condition [0x00007fc0e8b4d000] java.lang.Thread.State: WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007fc518c24578> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:750) "ReadOnlyZKClient-cod-1e3xsc06g8mjm-master1.exait3p.z1did1.b0.cloudera.site:2181,cod-1e3xsc06g8mjm-master0.exait3p.z1did1.b0.cloudera.site:2181,cod-1e3xsc06g8mjm-leader0.exait3p.z1did1.b0.cloudera.site:2181@0x57fa4a8d" #18802 daemon prio=5 os_prio=0 tid=0x00007fc09ca86000 nid=0x4fa waiting on condition [0x00007fc0e6f31000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007fc518c32578> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.DelayQueue.poll(DelayQueue.java:259) at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:323) at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$$Lambda$164/1053864765.run(Unknown Source) at java.lang.Thread.run(Thread.java:750) "AsyncConn Chore Service.Chore.1" #18801 daemon prio=5 os_prio=0 tid=0x00007fc09d263800 nid=0x4f9 waiting on condition [0x00007fc0f7938000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007fc518c2c720> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1093) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:750) "ReadOnlyZKClient-cod-1e3xsc06g8mjm-master1.exait3p.z1did1.b0.cloudera.site:2181,cod-1e3xsc06g8mjm-master0.exait3p.z1did1.b0.cloudera.site:2181,cod-1e3xsc06g8mjm-leader0.exait3p.z1did1.b0.cloudera.site:2181@0x0c37227a" #18784 daemon prio=5 os_prio=0 tid=0x00007fc09d16a800 nid=0x475 waiting on condition [0x00007fc0c2c1d000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007fc518c30578> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.DelayQueue.poll(DelayQueue.java:259) at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:323) at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$$Lambda$164/1053864765.run(Unknown Source) at java.lang.Thread.run(Thread.java:750) "AsyncConn Chore Service.Chore.1" #18783 daemon prio=5 os_prio=0 tid=0x00007fc09d169800 nid=0x474 waiting on condition [0x00007fc0e6426000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007fc518c24980> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1093) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:750) "java-sdk-progress-listener-callback-thread" #13578 daemon prio=5 os_prio=0 tid=0x00007fc0b601a000 nid=0x4d18 waiting on condition [0x00007fc0ee5a7000] java.lang.Thread.State: WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007fc50024bdf8> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:750) "group-cache-0" #6206 daemon prio=5 os_prio=0 tid=0x00007fc97f91b000 nid=0x6c7a waiting on condition [0x00007fc0ee9ab000] java.lang.Thread.State: WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007fc1c9e52d68> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1081) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:750) "RS-EventLoopGroup-1-16" #5656 daemon prio=10 os_prio=0 tid=0x00007fc986ecd800 nid=0x659e runnable [0x00007fc0ef1b3000] java.lang.Thread.State: RUNNABLE at org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) at org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:157) at org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:150) at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:290) at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:347) at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:989) at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) at java.lang.Thread.run(Thread.java:750) "RS-EventLoopGroup-1-15" #5655 daemon prio=10 os_prio=0 tid=0x00007fc984f30000 nid=0x659b runnable [0x00007fc0f6d2c000] java.lang.Thread.State: RUNNABLE at org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) at org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:157) at org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:150) at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:290) at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:347) at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:989) at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) at java.lang.Thread.run(Thread.java:750) "RS-EventLoopGroup-1-14" #5654 daemon prio=10 os_prio=0 tid=0x00007fc984a3d800 nid=0x6598 runnable [0x00007fc0ef0b2000] java.lang.Thread.State: RUNNABLE at org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) at org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:157) at org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:150) at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:290) at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:347) at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:989) at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) at java.lang.Thread.run(Thread.java:750) "RS-EventLoopGroup-1-13" #5653 daemon prio=10 os_prio=0 tid=0x00007fc986b41000 nid=0x6590 runnable [0x00007fc0ee8aa000] java.lang.Thread.State: RUNNABLE at org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) at org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:157) at org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:150) at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:290) at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:347) at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:989) at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) at java.lang.Thread.run(Thread.java:750) "RS-EventLoopGroup-1-12" #5652 daemon prio=10 os_prio=0 tid=0x0000558b142f8800 nid=0x658d runnable [0x00007fc1021dc000] java.lang.Thread.State: RUNNABLE at org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) at org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:157) at org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:150) at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:290) at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:347) at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:989) at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) at java.lang.Thread.run(Thread.java:750) "RS-EventLoopGroup-1-11" #5650 daemon prio=10 os_prio=0 tid=0x00007fc98641f000 nid=0x658b runnable [0x00007fc0eecae000] java.lang.Thread.State: RUNNABLE at org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) at org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:157) at org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:150) at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:290) at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:347) at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:989) at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) at java.lang.Thread.run(Thread.java:750) "ReadOnlyZKClient-cod-1e3xsc06g8mjm-master1.exait3p.z1did1.b0.cloudera.site:2181,cod-1e3xsc06g8mjm-master0.exait3p.z1did1.b0.cloudera.site:2181,cod-1e3xsc06g8mjm-leader0.exait3p.z1did1.b0.cloudera.site:2181@0x0182101e" #5645 daemon prio=5 os_prio=0 tid=0x00007fc986bb5800 nid=0x6585 waiting on condition [0x00007fc0eefb1000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007fc26ed23098> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.DelayQueue.poll(DelayQueue.java:259) at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:323) at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$$Lambda$164/1053864765.run(Unknown Source) at java.lang.Thread.run(Thread.java:750) "AsyncConn Chore Service.Chore.1" #5644 daemon prio=5 os_prio=0 tid=0x00007fc986bb4800 nid=0x6584 waiting on condition [0x00007fc0ee4a6000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007fc26ed36d78> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1093) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:750) "RS-EventLoopGroup-1-10" #5640 daemon prio=10 os_prio=0 tid=0x00007fc986bb4000 nid=0x6578 runnable [0x00007fc0edfa1000] java.lang.Thread.State: RUNNABLE at org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) at org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:157) at org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:150) at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:290) at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:347) at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:989) at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) at java.lang.Thread.run(Thread.java:750) "RS-EventLoopGroup-1-9" #5639 daemon prio=10 os_prio=0 tid=0x00007fc986e8c000 nid=0x6555 runnable [0x00007fc0f8a49000] java.lang.Thread.State: RUNNABLE at org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) at org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:157) at org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:150) at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:290) at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:347) at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:989) at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) at java.lang.Thread.run(Thread.java:750) "RS-EventLoopGroup-1-8" #5638 daemon prio=10 os_prio=0 tid=0x00007fc986bc8000 nid=0x654e runnable [0x00007fc0ec88a000] java.lang.Thread.State: RUNNABLE at org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) at org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:157) at org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:150) at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:290) at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:347) at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:989) at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) at java.lang.Thread.run(Thread.java:750) "RS-EventLoopGroup-1-7" #5635 daemon prio=10 os_prio=0 tid=0x00007fc986e8b800 nid=0x6546 runnable [0x00007fc0ec98b000] java.lang.Thread.State: RUNNABLE at org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) at org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:157) at org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:150) at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:290) at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:347) at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:989) at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) at java.lang.Thread.run(Thread.java:750) "RS-EventLoopGroup-1-6" #5634 daemon prio=10 os_prio=0 tid=0x00007fc986bbf000 nid=0x6537 runnable [0x00007fc0f04c4000] java.lang.Thread.State: RUNNABLE at org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) at org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:157) at org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:150) at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:290) at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:347) at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:989) at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) at java.lang.Thread.run(Thread.java:750) "RS_COMPACTED_FILES_DISCHARGER-regionserver/cod-1e3xsc06g8mjm-worker124:16020-9" #5613 daemon prio=5 os_prio=0 tid=0x00007fc986585800 nid=0x62a5 waiting on condition [0x00007fc0ed698000] java.lang.Thread.State: WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007fc1dc35dea0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:750) "RS_COMPACTED_FILES_DISCHARGER-regionserver/cod-1e3xsc06g8mjm-worker124:16020-8" #5612 daemon prio=5 os_prio=0 tid=0x00007fc986584800 nid=0x62a4 waiting on condition [0x00007fc0ed597000] java.lang.Thread.State: WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007fc1dc35dea0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:750) "RS_COMPACTED_FILES_DISCHARGER-regionserver/cod-1e3xsc06g8mjm-worker124:16020-7" #5611 daemon prio=5 os_prio=0 tid=0x00007fc9865a8000 nid=0x62a3 waiting on condition [0x00007fc0ecf91000] java.lang.Thread.State: WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007fc1dc35dea0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:750) "RS_COMPACTED_FILES_DISCHARGER-regionserver/cod-1e3xsc06g8mjm-worker124:16020-6" #5610 daemon prio=5 os_prio=0 tid=0x00007fc9865a6800 nid=0x62a2 waiting on condition [0x00007fc0ed496000] java.lang.Thread.State: WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007fc1dc35dea0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:750) "RS_COMPACTED_FILES_DISCHARGER-regionserver/cod-1e3xsc06g8mjm-worker124:16020-5" #5609 daemon prio=5 os_prio=0 tid=0x00007fc9865a6000 nid=0x62a1 waiting on condition [0x00007fc0ed193000] java.lang.Thread.State: WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007fc1dc35dea0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:750) "RS_COMPACTED_FILES_DISCHARGER-regionserver/cod-1e3xsc06g8mjm-worker124:16020-4" #5608 daemon prio=5 os_prio=0 tid=0x00007fc9865a5000 nid=0x62a0 waiting on condition [0x00007fc0ecc8e000] java.lang.Thread.State: WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007fc1dc35dea0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:750) "RS_COMPACTED_FILES_DISCHARGER-regionserver/cod-1e3xsc06g8mjm-worker124:16020-3" #5607 daemon prio=5 os_prio=0 tid=0x00007fc9865a4800 nid=0x629f waiting on condition [0x00007fc0eca8c000] java.lang.Thread.State: WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007fc1dc35dea0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:750) "RS_COMPACTED_FILES_DISCHARGER-regionserver/cod-1e3xsc06g8mjm-worker124:16020-2" #5606 daemon prio=5 os_prio=0 tid=0x00007fc9865a3800 nid=0x629e waiting on condition [0x00007fc108719000] java.lang.Thread.State: WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007fc1dc35dea0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:750) "RS_COMPACTED_FILES_DISCHARGER-regionserver/cod-1e3xsc06g8mjm-worker124:16020-1" #5605 daemon prio=5 os_prio=0 tid=0x00007fc986554800 nid=0x629d waiting on condition [0x00007fc108b1b000] java.lang.Thread.State: WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007fc1dc35dea0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:750) "RS_COMPACTED_FILES_DISCHARGER-regionserver/cod-1e3xsc06g8mjm-worker124:16020-0" #5604 daemon prio=5 os_prio=0 tid=0x00007fc98656a000 nid=0x629c waiting on condition [0x00007fc0ecd8f000] java.lang.Thread.State: WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007fc1dc35dea0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:750) "Log-Archiver-0" #5173 daemon prio=5 os_prio=0 tid=0x00007fc11de5b800 nid=0x3553 waiting on condition [0x00007fc0ecb8d000] java.lang.Thread.State: WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007fc1dc97d160> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:750) "ReadOnlyZKClient-cod-1e3xsc06g8mjm-master1.exait3p.z1did1.b0.cloudera.site:2181,cod-1e3xsc06g8mjm-master0.exait3p.z1did1.b0.cloudera.site:2181,cod-1e3xsc06g8mjm-leader0.exait3p.z1did1.b0.cloudera.site:2181@0x01a63428" #5072 daemon prio=5 os_prio=0 tid=0x00007fc11def9800 nid=0x2c34 waiting on condition [0x00007fc0ed99b000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007fc207614580> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.DelayQueue.poll(DelayQueue.java:259) at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:323) at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$$Lambda$164/1053864765.run(Unknown Source) at java.lang.Thread.run(Thread.java:750) "ReadOnlyZKClient-cod-1e3xsc06g8mjm-master1.exait3p.z1did1.b0.cloudera.site:2181,cod-1e3xsc06g8mjm-master0.exait3p.z1did1.b0.cloudera.site:2181,cod-1e3xsc06g8mjm-leader0.exait3p.z1did1.b0.cloudera.site:2181@0x5dad7ab9" #5071 daemon prio=5 os_prio=0 tid=0x00007fc11def9000 nid=0x2c33 waiting on condition [0x00007fc0edb9d000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007fc20761c580> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.DelayQueue.poll(DelayQueue.java:259) at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:323) at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$$Lambda$164/1053864765.run(Unknown Source) at java.lang.Thread.run(Thread.java:750) "AsyncConn Chore Service.Chore.1" #5070 daemon prio=5 os_prio=0 tid=0x00007fc11def8000 nid=0x2c32 waiting on condition [0x00007fc0edc9e000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007fc207614780> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1093) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:750) "AsyncConn Chore Service.Chore.1" #5069 daemon prio=5 os_prio=0 tid=0x00007fc11def7000 nid=0x2c31 waiting on condition [0x00007fc0edd9f000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007fc20761c780> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1093) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:750) "ReadOnlyZKClient-cod-1e3xsc06g8mjm-master1.exait3p.z1did1.b0.cloudera.site:2181,cod-1e3xsc06g8mjm-master0.exait3p.z1did1.b0.cloudera.site:2181,cod-1e3xsc06g8mjm-leader0.exait3p.z1did1.b0.cloudera.site:2181@0x446d0616" #5026 daemon prio=5 os_prio=0 tid=0x00007fc97cc61000 nid=0x2b90 waiting on condition [0x00007fc0f0dcd000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007fc813b7a998> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.DelayQueue.poll(DelayQueue.java:259) at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:323) at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$$Lambda$164/1053864765.run(Unknown Source) at java.lang.Thread.run(Thread.java:750) "ReadOnlyZKClient-cod-1e3xsc06g8mjm-master1.exait3p.z1did1.b0.cloudera.site:2181,cod-1e3xsc06g8mjm-master0.exait3p.z1did1.b0.cloudera.site:2181,cod-1e3xsc06g8mjm-leader0.exait3p.z1did1.b0.cloudera.site:2181@0x1263bc0f" #5025 daemon prio=5 os_prio=0 tid=0x00007fc11e11b000 nid=0x2b8f waiting on condition [0x00007fc0f0fcf000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007fc813b78998> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.DelayQueue.poll(DelayQueue.java:259) at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:323) at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$$Lambda$164/1053864765.run(Unknown Source) at java.lang.Thread.run(Thread.java:750) "AsyncConn Chore Service.Chore.1" #5024 daemon prio=5 os_prio=0 tid=0x00007fc97cc60000 nid=0x2b8e waiting on condition [0x00007fc0f12d2000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007fc813b7ab98> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1093) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:750) "AsyncConn Chore Service.Chore.1" #5023 daemon prio=5 os_prio=0 tid=0x00007fc11defe000 nid=0x2b8d waiting on condition [0x00007fc0f21e1000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007fc813b78b98> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1093) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:750) "ReadOnlyZKClient-cod-1e3xsc06g8mjm-master1.exait3p.z1did1.b0.cloudera.site:2181,cod-1e3xsc06g8mjm-master0.exait3p.z1did1.b0.cloudera.site:2181,cod-1e3xsc06g8mjm-leader0.exait3p.z1did1.b0.cloudera.site:2181@0x1dca0dc9" #5017 daemon prio=5 os_prio=0 tid=0x00007fc984da0000 nid=0x2b85 waiting on condition [0x00007fc0f6423000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007fc813b7add8> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.DelayQueue.poll(DelayQueue.java:259) at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:323) at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$$Lambda$164/1053864765.run(Unknown Source) at java.lang.Thread.run(Thread.java:750) "AsyncConn Chore Service.Chore.1" #5016 daemon prio=5 os_prio=0 tid=0x00007fc984d9f000 nid=0x2b84 waiting on condition [0x00007fc0f6827000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007fc813b78dd8> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1093) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:750) "ReadOnlyZKClient-cod-1e3xsc06g8mjm-master1.exait3p.z1did1.b0.cloudera.site:2181,cod-1e3xsc06g8mjm-master0.exait3p.z1did1.b0.cloudera.site:2181,cod-1e3xsc06g8mjm-leader0.exait3p.z1did1.b0.cloudera.site:2181@0x333a7fe7" #5004 daemon prio=5 os_prio=0 tid=0x00007fc984b4e800 nid=0x2b75 waiting on condition [0x00007fc0fbd7a000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007fc813b7afd8> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.DelayQueue.poll(DelayQueue.java:259) at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:323) at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$$Lambda$164/1053864765.run(Unknown Source) at java.lang.Thread.run(Thread.java:750) "AsyncConn Chore Service.Chore.1" #5003 daemon prio=5 os_prio=0 tid=0x00007fc984b51800 nid=0x2b74 waiting on condition [0x00007fc0fcf8c000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007fc813b79018> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1093) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:750) "ReadOnlyZKClient-cod-1e3xsc06g8mjm-master1.exait3p.z1did1.b0.cloudera.site:2181,cod-1e3xsc06g8mjm-master0.exait3p.z1did1.b0.cloudera.site:2181,cod-1e3xsc06g8mjm-leader0.exait3p.z1did1.b0.cloudera.site:2181@0x0be4be12" #5000 daemon prio=5 os_prio=0 tid=0x00007fc984b50800 nid=0x2b71 waiting on condition [0x00007fc0fe9a6000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007fc813b7b1d8> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.DelayQueue.poll(DelayQueue.java:259) at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:323) at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$$Lambda$164/1053864765.run(Unknown Source) at java.lang.Thread.run(Thread.java:750) "AsyncConn Chore Service.Chore.1" #4999 daemon prio=5 os_prio=0 tid=0x00007fc984b54800 nid=0x2b70 waiting on condition [0x00007fc0feca9000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007fc813b79258> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1093) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:750) "ReadOnlyZKClient-cod-1e3xsc06g8mjm-master1.exait3p.z1did1.b0.cloudera.site:2181,cod-1e3xsc06g8mjm-master0.exait3p.z1did1.b0.cloudera.site:2181,cod-1e3xsc06g8mjm-leader0.exait3p.z1did1.b0.cloudera.site:2181@0x79a6905c" #4989 daemon prio=5 os_prio=0 tid=0x00007fc11defd800 nid=0x2b62 waiting on condition [0x00007fc0e18db000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007fc813b7b3d8> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.DelayQueue.poll(DelayQueue.java:259) at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:323) at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$$Lambda$164/1053864765.run(Unknown Source) at java.lang.Thread.run(Thread.java:750) "AsyncConn Chore Service.Chore.1" #4988 daemon prio=5 os_prio=0 tid=0x00007fc11e17f000 nid=0x2b61 waiting on condition [0x00007fc0e0ccf000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007fc813b79498> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1093) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:750) "ReadOnlyZKClient-cod-1e3xsc06g8mjm-master1.exait3p.z1did1.b0.cloudera.site:2181,cod-1e3xsc06g8mjm-master0.exait3p.z1did1.b0.cloudera.site:2181,cod-1e3xsc06g8mjm-leader0.exait3p.z1did1.b0.cloudera.site:2181@0x729b84e0" #4970 daemon prio=5 os_prio=0 tid=0x00007fc11e11d800 nid=0x2b45 waiting on condition [0x00007fc0fda97000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007fc813b7b5d8> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.DelayQueue.poll(DelayQueue.java:259) at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:323) at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$$Lambda$164/1053864765.run(Unknown Source) at java.lang.Thread.run(Thread.java:750) "AsyncConn Chore Service.Chore.1" #4969 daemon prio=5 os_prio=0 tid=0x00007fc11d14c800 nid=0x2b44 waiting on condition [0x00007fc0e08cb000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007fc813b796d8> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1093) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:750) "ReadOnlyZKClient-cod-1e3xsc06g8mjm-master1.exait3p.z1did1.b0.cloudera.site:2181,cod-1e3xsc06g8mjm-master0.exait3p.z1did1.b0.cloudera.site:2181,cod-1e3xsc06g8mjm-leader0.exait3p.z1did1.b0.cloudera.site:2181@0x625e15ae" #4959 daemon prio=5 os_prio=0 tid=0x00007fc97cca5000 nid=0x2b39 waiting on condition [0x00007fc0e06c9000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007fc813b7b7d8> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.DelayQueue.poll(DelayQueue.java:259) at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:323) at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$$Lambda$164/1053864765.run(Unknown Source) at java.lang.Thread.run(Thread.java:750) "AsyncConn Chore Service.Chore.1" #4958 daemon prio=5 os_prio=0 tid=0x00007fc97cca4800 nid=0x2b38 waiting on condition [0x00007fc0e6e30000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007fc813b79918> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1093) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:750) "ReadOnlyZKClient-cod-1e3xsc06g8mjm-master1.exait3p.z1did1.b0.cloudera.site:2181,cod-1e3xsc06g8mjm-master0.exait3p.z1did1.b0.cloudera.site:2181,cod-1e3xsc06g8mjm-leader0.exait3p.z1did1.b0.cloudera.site:2181@0x4bd61370" #4952 daemon prio=5 os_prio=0 tid=0x0000558b141ec800 nid=0x2b32 waiting on condition [0x00007fc0fd390000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007fc813b79b58> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.DelayQueue.poll(DelayQueue.java:259) at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:323) at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$$Lambda$164/1053864765.run(Unknown Source) at java.lang.Thread.run(Thread.java:750) "AsyncConn Chore Service.Chore.1" #4951 daemon prio=5 os_prio=0 tid=0x0000558b1492f000 nid=0x2b31 waiting on condition [0x00007fc0f38f8000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007fc813b79d58> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1093) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:750) "RS-EventLoopGroup-1-5" #4942 daemon prio=10 os_prio=0 tid=0x00007fc985374000 nid=0x2b27 runnable [0x00007fc0e6527000] java.lang.Thread.State: RUNNABLE at org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) at org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:157) at org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:150) at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:290) at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:347) at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:989) at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) at java.lang.Thread.run(Thread.java:750) "ReadOnlyZKClient-cod-1e3xsc06g8mjm-master1.exait3p.z1did1.b0.cloudera.site:2181,cod-1e3xsc06g8mjm-master0.exait3p.z1did1.b0.cloudera.site:2181,cod-1e3xsc06g8mjm-leader0.exait3p.z1did1.b0.cloudera.site:2181@0x4cbef3cd" #4894 daemon prio=5 os_prio=0 tid=0x0000558b142a6000 nid=0x290c waiting on condition [0x00007fc0eb577000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007fc2018dc578> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.DelayQueue.poll(DelayQueue.java:259) at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:323) at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$$Lambda$164/1053864765.run(Unknown Source) at java.lang.Thread.run(Thread.java:750) "ReadOnlyZKClient-cod-1e3xsc06g8mjm-master1.exait3p.z1did1.b0.cloudera.site:2181,cod-1e3xsc06g8mjm-master0.exait3p.z1did1.b0.cloudera.site:2181,cod-1e3xsc06g8mjm-leader0.exait3p.z1did1.b0.cloudera.site:2181@0x577ceae3" #4893 daemon prio=5 os_prio=0 tid=0x00007fc984f8e000 nid=0x290b waiting on condition [0x00007fc0eb87a000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007fc2018dc778> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.DelayQueue.poll(DelayQueue.java:259) at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:323) at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$$Lambda$164/1053864765.run(Unknown Source) at java.lang.Thread.run(Thread.java:750) "AsyncConn Chore Service.Chore.1" #4892 daemon prio=5 os_prio=0 tid=0x0000558b142fa000 nid=0x290a waiting on condition [0x00007fc0eb97b000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007fc2018da578> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1093) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:750) "AsyncConn Chore Service.Chore.1" #4891 daemon prio=5 os_prio=0 tid=0x00007fc984121800 nid=0x2909 waiting on condition [0x00007fc0ece90000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007fc2018dc978> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1093) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:750) "ReadOnlyZKClient-cod-1e3xsc06g8mjm-master1.exait3p.z1did1.b0.cloudera.site:2181,cod-1e3xsc06g8mjm-master0.exait3p.z1did1.b0.cloudera.site:2181,cod-1e3xsc06g8mjm-leader0.exait3p.z1did1.b0.cloudera.site:2181@0x738d2d28" #4874 daemon prio=5 os_prio=0 tid=0x00007fc985363800 nid=0x28de waiting on condition [0x00007fc0e1de0000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007fc20127eb80> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.DelayQueue.poll(DelayQueue.java:259) at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:323) at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$$Lambda$164/1053864765.run(Unknown Source) at java.lang.Thread.run(Thread.java:750) "AsyncConn Chore Service.Chore.1" #4873 daemon prio=5 os_prio=0 tid=0x00007fc98518f000 nid=0x28dd waiting on condition [0x00007fc0e0acd000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007fc20127cb80> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1093) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:750) "ReadOnlyZKClient-cod-1e3xsc06g8mjm-master1.exait3p.z1did1.b0.cloudera.site:2181,cod-1e3xsc06g8mjm-master0.exait3p.z1did1.b0.cloudera.site:2181,cod-1e3xsc06g8mjm-leader0.exait3p.z1did1.b0.cloudera.site:2181@0x7a128649" #4856 daemon prio=5 os_prio=0 tid=0x00007fc11df32000 nid=0x28bd waiting on condition [0x00007fc0e1ee1000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007fc2009f2d78> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.DelayQueue.poll(DelayQueue.java:259) at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:323) at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$$Lambda$164/1053864765.run(Unknown Source) at java.lang.Thread.run(Thread.java:750) "ReadOnlyZKClient-cod-1e3xsc06g8mjm-master1.exait3p.z1did1.b0.cloudera.site:2181,cod-1e3xsc06g8mjm-master0.exait3p.z1did1.b0.cloudera.site:2181,cod-1e3xsc06g8mjm-leader0.exait3p.z1did1.b0.cloudera.site:2181@0x2f7e4b0e" #4855 daemon prio=5 os_prio=0 tid=0x00007fc97caa2800 nid=0x28bc waiting on condition [0x00007fc0e21e4000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007fc2009f0d78> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.DelayQueue.poll(DelayQueue.java:259) at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:323) at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$$Lambda$164/1053864765.run(Unknown Source) at java.lang.Thread.run(Thread.java:750) "AsyncConn Chore Service.Chore.1" #4854 daemon prio=5 os_prio=0 tid=0x00007fc11df33000 nid=0x28bb waiting on condition [0x00007fc0e20e3000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007fc2009f0f78> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1093) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:750) "AsyncConn Chore Service.Chore.1" #4853 daemon prio=5 os_prio=0 tid=0x00007fc97de16000 nid=0x28ba waiting on condition [0x00007fc0e1cdf000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007fc2009f11b8> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1093) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:750) "ReadOnlyZKClient-cod-1e3xsc06g8mjm-master1.exait3p.z1did1.b0.cloudera.site:2181,cod-1e3xsc06g8mjm-master0.exait3p.z1did1.b0.cloudera.site:2181,cod-1e3xsc06g8mjm-leader0.exait3p.z1did1.b0.cloudera.site:2181@0x5858e6b7" #4840 daemon prio=5 os_prio=0 tid=0x00007fc97d859000 nid=0x2891 waiting on condition [0x00007fc0e2bee000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007fc2009f13f8> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.DelayQueue.poll(DelayQueue.java:259) at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:323) at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$$Lambda$164/1053864765.run(Unknown Source) at java.lang.Thread.run(Thread.java:750) "AsyncConn Chore Service.Chore.1" #4839 daemon prio=5 os_prio=0 tid=0x00007fc97caa4000 nid=0x2890 waiting on condition [0x00007fc0f6e2d000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007fc2009f15f8> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1093) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:750) "ReadOnlyZKClient-cod-1e3xsc06g8mjm-master1.exait3p.z1did1.b0.cloudera.site:2181,cod-1e3xsc06g8mjm-master0.exait3p.z1did1.b0.cloudera.site:2181,cod-1e3xsc06g8mjm-leader0.exait3p.z1did1.b0.cloudera.site:2181@0x4c87c771" #4829 daemon prio=5 os_prio=0 tid=0x00007fc11de43800 nid=0x287e waiting on condition [0x00007fc0f11d1000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007fc2002efd78> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.DelayQueue.poll(DelayQueue.java:259) at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:323) at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$$Lambda$164/1053864765.run(Unknown Source) at java.lang.Thread.run(Thread.java:750) "AsyncConn Chore Service.Chore.1" #4828 daemon prio=5 os_prio=0 tid=0x00007fc11dcb7000 nid=0x287d waiting on condition [0x00007fc0f5b1a000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007fc2002f1d78> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1093) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:750) "ReadOnlyZKClient-cod-1e3xsc06g8mjm-master1.exait3p.z1did1.b0.cloudera.site:2181,cod-1e3xsc06g8mjm-master0.exait3p.z1did1.b0.cloudera.site:2181,cod-1e3xsc06g8mjm-leader0.exait3p.z1did1.b0.cloudera.site:2181@0x46e42e2e" #4800 daemon prio=5 os_prio=0 tid=0x00007fc98454d800 nid=0x284c waiting on condition [0x00007fc0f9855000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007fc1ffab7f78> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.DelayQueue.poll(DelayQueue.java:259) at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:323) at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$$Lambda$164/1053864765.run(Unknown Source) at java.lang.Thread.run(Thread.java:750) "AsyncConn Chore Service.Chore.1" #4799 daemon prio=5 os_prio=0 tid=0x00007fc985548000 nid=0x284b waiting on condition [0x00007fc0e7335000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007fc1ffab9f78> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1093) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:750) "ReadOnlyZKClient-cod-1e3xsc06g8mjm-master1.exait3p.z1did1.b0.cloudera.site:2181,cod-1e3xsc06g8mjm-master0.exait3p.z1did1.b0.cloudera.site:2181,cod-1e3xsc06g8mjm-leader0.exait3p.z1did1.b0.cloudera.site:2181@0x69562906" #4792 daemon prio=5 os_prio=0 tid=0x00007fc11e405800 nid=0x2840 waiting on condition [0x00007fc0e692b000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007fc1ffab5f78> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.DelayQueue.poll(DelayQueue.java:259) at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:323) at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$$Lambda$164/1053864765.run(Unknown Source) at java.lang.Thread.run(Thread.java:750) "AsyncConn Chore Service.Chore.1" #4791 daemon prio=5 os_prio=0 tid=0x00007fc11e17c800 nid=0x283f waiting on condition [0x00007fc0e6628000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007fc1ffab8178> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1093) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:750) "ReadOnlyZKClient-cod-1e3xsc06g8mjm-master1.exait3p.z1did1.b0.cloudera.site:2181,cod-1e3xsc06g8mjm-master0.exait3p.z1did1.b0.cloudera.site:2181,cod-1e3xsc06g8mjm-leader0.exait3p.z1did1.b0.cloudera.site:2181@0x5ca61b8f" #4757 daemon prio=5 os_prio=0 tid=0x0000558b14a1c000 nid=0x27e7 waiting on condition [0x00007fc0ea264000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007fc1feca8b78> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.DelayQueue.poll(DelayQueue.java:259) at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:323) at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$$Lambda$164/1053864765.run(Unknown Source) at java.lang.Thread.run(Thread.java:750) "AsyncConn Chore Service.Chore.1" #4756 daemon prio=5 os_prio=0 tid=0x0000558b14600000 nid=0x27e6 waiting on condition [0x00007fc0ea365000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007fc1feca6b78> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1093) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:750) "ReadOnlyZKClient-cod-1e3xsc06g8mjm-master1.exait3p.z1did1.b0.cloudera.site:2181,cod-1e3xsc06g8mjm-master0.exait3p.z1did1.b0.cloudera.site:2181,cod-1e3xsc06g8mjm-leader0.exait3p.z1did1.b0.cloudera.site:2181@0x070fbee6" #4745 daemon prio=5 os_prio=0 tid=0x0000558b13fc8800 nid=0x27d3 waiting on condition [0x00007fc0f39f9000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007fc1fecaab78> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.DelayQueue.poll(DelayQueue.java:259) at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:323) at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$$Lambda$164/1053864765.run(Unknown Source) at java.lang.Thread.run(Thread.java:750) "AsyncConn Chore Service.Chore.1" #4744 daemon prio=5 os_prio=0 tid=0x0000558b14258000 nid=0x27d2 waiting on condition [0x00007fc0eb375000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007fc1feca8d78> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1093) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:750) "ReadOnlyZKClient-cod-1e3xsc06g8mjm-master1.exait3p.z1did1.b0.cloudera.site:2181,cod-1e3xsc06g8mjm-master0.exait3p.z1did1.b0.cloudera.site:2181,cod-1e3xsc06g8mjm-leader0.exait3p.z1did1.b0.cloudera.site:2181@0x243968a9" #4731 daemon prio=5 os_prio=0 tid=0x00007fc11dc40000 nid=0x2738 waiting on condition [0x00007fc0e03c6000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007fc1fecaad78> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.DelayQueue.poll(DelayQueue.java:259) at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:323) at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$$Lambda$164/1053864765.run(Unknown Source) at java.lang.Thread.run(Thread.java:750) "AsyncConn Chore Service.Chore.1" #4730 daemon prio=5 os_prio=0 tid=0x00007fc11dc3f800 nid=0x2737 waiting on condition [0x00007fc0e05c8000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007fc1fecaaf78> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1093) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:750) "ReadOnlyZKClient-cod-1e3xsc06g8mjm-master1.exait3p.z1did1.b0.cloudera.site:2181,cod-1e3xsc06g8mjm-master0.exait3p.z1did1.b0.cloudera.site:2181,cod-1e3xsc06g8mjm-leader0.exait3p.z1did1.b0.cloudera.site:2181@0x6cfa920d" #4715 daemon prio=5 os_prio=0 tid=0x00007fc11df3b000 nid=0x271b waiting on condition [0x00007fc0e0dd0000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007fc1fe49fb78> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.DelayQueue.poll(DelayQueue.java:259) at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:323) at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$$Lambda$164/1053864765.run(Unknown Source) at java.lang.Thread.run(Thread.java:750) "AsyncConn Chore Service.Chore.1" #4714 daemon prio=5 os_prio=0 tid=0x00007fc11df3a000 nid=0x271a waiting on condition [0x00007fc0e28eb000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007fc1fe49db78> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1093) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:750) "ReadOnlyZKClient-cod-1e3xsc06g8mjm-master1.exait3p.z1did1.b0.cloudera.site:2181,cod-1e3xsc06g8mjm-master0.exait3p.z1did1.b0.cloudera.site:2181,cod-1e3xsc06g8mjm-leader0.exait3p.z1did1.b0.cloudera.site:2181@0x3d9c33d4" #4699 daemon prio=5 os_prio=0 tid=0x00007fc984940000 nid=0x2706 waiting on condition [0x00007fc0e14d7000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007fc1fe4a1b78> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.DelayQueue.poll(DelayQueue.java:259) at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:323) at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$$Lambda$164/1053864765.run(Unknown Source) at java.lang.Thread.run(Thread.java:750) "AsyncConn Chore Service.Chore.1" #4698 daemon prio=5 os_prio=0 tid=0x00007fc98493e000 nid=0x2705 waiting on condition [0x00007fc0e15d8000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007fc1fe49fd78> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1093) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:750) "ReadOnlyZKClient-cod-1e3xsc06g8mjm-master1.exait3p.z1did1.b0.cloudera.site:2181,cod-1e3xsc06g8mjm-master0.exait3p.z1did1.b0.cloudera.site:2181,cod-1e3xsc06g8mjm-leader0.exait3p.z1did1.b0.cloudera.site:2181@0x3a356343" #4685 daemon prio=5 os_prio=0 tid=0x00007fc98512c000 nid=0x26f1 waiting on condition [0x00007fc0e22e5000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007fc1fdb48578> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.DelayQueue.poll(DelayQueue.java:259) at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:323) at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$$Lambda$164/1053864765.run(Unknown Source) at java.lang.Thread.run(Thread.java:750) "AsyncConn Chore Service.Chore.1" #4684 daemon prio=5 os_prio=0 tid=0x00007fc985130800 nid=0x26f0 waiting on condition [0x00007fc0e23e6000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007fc1fdb44578> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1093) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:750) "ReadOnlyZKClient-cod-1e3xsc06g8mjm-master1.exait3p.z1did1.b0.cloudera.site:2181,cod-1e3xsc06g8mjm-master0.exait3p.z1did1.b0.cloudera.site:2181,cod-1e3xsc06g8mjm-leader0.exait3p.z1did1.b0.cloudera.site:2181@0x146ed4b7" #4681 daemon prio=5 os_prio=0 tid=0x00007fc98512e800 nid=0x26ed waiting on condition [0x00007fc0e26e9000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007fc1fdb46578> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.DelayQueue.poll(DelayQueue.java:259) at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:323) at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$$Lambda$164/1053864765.run(Unknown Source) at java.lang.Thread.run(Thread.java:750) "AsyncConn Chore Service.Chore.1" #4680 daemon prio=5 os_prio=0 tid=0x00007fc98512d000 nid=0x26ec waiting on condition [0x00007fc0e27ea000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007fc1fdb4a578> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1093) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:750) "ReadOnlyZKClient-cod-1e3xsc06g8mjm-master1.exait3p.z1did1.b0.cloudera.site:2181,cod-1e3xsc06g8mjm-master0.exait3p.z1did1.b0.cloudera.site:2181,cod-1e3xsc06g8mjm-leader0.exait3p.z1did1.b0.cloudera.site:2181@0x657da81a" #4670 daemon prio=5 os_prio=0 tid=0x00007fc11e3ec000 nid=0x26c1 waiting on condition [0x00007fc0e2df0000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007fc1fdb48778> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.DelayQueue.poll(DelayQueue.java:259) at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:323) at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$$Lambda$164/1053864765.run(Unknown Source) at java.lang.Thread.run(Thread.java:750) "AsyncConn Chore Service.Chore.1" #4669 daemon prio=5 os_prio=0 tid=0x00007fc11e3f8800 nid=0x26c0 waiting on condition [0x00007fc0e2ef1000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007fc1fdb447b8> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1093) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:750) "ReadOnlyZKClient-cod-1e3xsc06g8mjm-master1.exait3p.z1did1.b0.cloudera.site:2181,cod-1e3xsc06g8mjm-master0.exait3p.z1did1.b0.cloudera.site:2181,cod-1e3xsc06g8mjm-leader0.exait3p.z1did1.b0.cloudera.site:2181@0x39c745fa" #4660 daemon prio=5 os_prio=0 tid=0x0000558b1494e800 nid=0x26b3 waiting on condition [0x00007fc0f6120000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007fc1fdb48978> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.DelayQueue.poll(DelayQueue.java:259) at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:323) at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$$Lambda$164/1053864765.run(Unknown Source) at java.lang.Thread.run(Thread.java:750) "AsyncConn Chore Service.Chore.1" #4659 daemon prio=5 os_prio=0 tid=0x0000558b1416d000 nid=0x26b2 waiting on condition [0x00007fc109323000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007fc1fdb4a7b8> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1093) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:750) "ReadOnlyZKClient-cod-1e3xsc06g8mjm-master1.exait3p.z1did1.b0.cloudera.site:2181,cod-1e3xsc06g8mjm-master0.exait3p.z1did1.b0.cloudera.site:2181,cod-1e3xsc06g8mjm-leader0.exait3p.z1did1.b0.cloudera.site:2181@0x1921bb0b" #4645 daemon prio=5 os_prio=0 tid=0x00007fc11dc91000 nid=0x269f waiting on condition [0x00007fc0feeab000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007fc1fc929978> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.DelayQueue.poll(DelayQueue.java:259) at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:323) at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$$Lambda$164/1053864765.run(Unknown Source) at java.lang.Thread.run(Thread.java:750) "AsyncConn Chore Service.Chore.1" #4644 daemon prio=5 os_prio=0 tid=0x00007fc11dc74000 nid=0x269e waiting on condition [0x00007fc1022dd000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007fc1fc927978> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1093) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:750) "ReadOnlyZKClient-cod-1e3xsc06g8mjm-master1.exait3p.z1did1.b0.cloudera.site:2181,cod-1e3xsc06g8mjm-master0.exait3p.z1did1.b0.cloudera.site:2181,cod-1e3xsc06g8mjm-leader0.exait3p.z1did1.b0.cloudera.site:2181@0x106ff8c6" #4632 daemon prio=5 os_prio=0 tid=0x00007fc984e98000 nid=0x268e waiting on condition [0x00007fc0f6524000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007fc1fc92d978> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.DelayQueue.poll(DelayQueue.java:259) at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:323) at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$$Lambda$164/1053864765.run(Unknown Source) at java.lang.Thread.run(Thread.java:750) "AsyncConn Chore Service.Chore.1" #4631 daemon prio=5 os_prio=0 tid=0x00007fc98411f000 nid=0x268d waiting on condition [0x00007fc0f37f7000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007fc1fc927bb8> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1093) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:750) "ReadOnlyZKClient-cod-1e3xsc06g8mjm-master1.exait3p.z1did1.b0.cloudera.site:2181,cod-1e3xsc06g8mjm-master0.exait3p.z1did1.b0.cloudera.site:2181,cod-1e3xsc06g8mjm-leader0.exait3p.z1did1.b0.cloudera.site:2181@0x2a00ffde" #4626 daemon prio=5 os_prio=0 tid=0x00007fc97db84000 nid=0x2688 waiting on condition [0x00007fc0f8c4b000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007fc1fc929b78> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.DelayQueue.poll(DelayQueue.java:259) at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:323) at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$$Lambda$164/1053864765.run(Unknown Source) at java.lang.Thread.run(Thread.java:750) "ReadOnlyZKClient-cod-1e3xsc06g8mjm-master1.exait3p.z1did1.b0.cloudera.site:2181,cod-1e3xsc06g8mjm-master0.exait3p.z1did1.b0.cloudera.site:2181,cod-1e3xsc06g8mjm-leader0.exait3p.z1did1.b0.cloudera.site:2181@0x1673f94a" #4625 daemon prio=5 os_prio=0 tid=0x0000558b14603000 nid=0x2687 waiting on condition [0x00007fc0e5719000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007fc1fc92db78> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.DelayQueue.poll(DelayQueue.java:259) at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:323) at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$$Lambda$164/1053864765.run(Unknown Source) at java.lang.Thread.run(Thread.java:750) "AsyncConn Chore Service.Chore.1" #4624 daemon prio=5 os_prio=0 tid=0x00007fc97c253000 nid=0x2686 waiting on condition [0x00007fc1010cb000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007fc1fc929d78> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1093) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:750) "AsyncConn Chore Service.Chore.1" #4623 daemon prio=5 os_prio=0 tid=0x0000558b1445e000 nid=0x2685 waiting on condition [0x00007fc0e5517000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007fc1fc927df8> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1093) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:750) "ReadOnlyZKClient-cod-1e3xsc06g8mjm-master1.exait3p.z1did1.b0.cloudera.site:2181,cod-1e3xsc06g8mjm-master0.exait3p.z1did1.b0.cloudera.site:2181,cod-1e3xsc06g8mjm-leader0.exait3p.z1did1.b0.cloudera.site:2181@0x06bfed53" #4613 daemon prio=5 os_prio=0 tid=0x00007fc11e176800 nid=0x2676 waiting on condition [0x00007fc0f702f000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007fc1fc929fb8> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.DelayQueue.poll(DelayQueue.java:259) at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:323) at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$$Lambda$164/1053864765.run(Unknown Source) at java.lang.Thread.run(Thread.java:750) "AsyncConn Chore Service.Chore.1" #4612 daemon prio=5 os_prio=0 tid=0x00007fc11dc90800 nid=0x2675 waiting on condition [0x00007fc0f7130000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007fc1fc928038> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1093) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:750) "ReadOnlyZKClient-cod-1e3xsc06g8mjm-master1.exait3p.z1did1.b0.cloudera.site:2181,cod-1e3xsc06g8mjm-master0.exait3p.z1did1.b0.cloudera.site:2181,cod-1e3xsc06g8mjm-leader0.exait3p.z1did1.b0.cloudera.site:2181@0x2f728d3a" #4606 daemon prio=5 os_prio=0 tid=0x00007fc9845fa800 nid=0x266d waiting on condition [0x00007fc0e38fb000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007fc1fc92a1b8> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.DelayQueue.poll(DelayQueue.java:259) at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:323) at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$$Lambda$164/1053864765.run(Unknown Source) at java.lang.Thread.run(Thread.java:750) "AsyncConn Chore Service.Chore.1" #4605 daemon prio=5 os_prio=0 tid=0x00007fc9845f8800 nid=0x266c waiting on condition [0x00007fc0e39fc000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007fc1fc928278> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1093) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:750) "ReadOnlyZKClient-cod-1e3xsc06g8mjm-master1.exait3p.z1did1.b0.cloudera.site:2181,cod-1e3xsc06g8mjm-master0.exait3p.z1did1.b0.cloudera.site:2181,cod-1e3xsc06g8mjm-leader0.exait3p.z1did1.b0.cloudera.site:2181@0x610656d8" #4587 daemon prio=5 os_prio=0 tid=0x00007fc984d2b800 nid=0x2658 waiting on condition [0x00007fc0e480a000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007fc1fc9284b8> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.DelayQueue.poll(DelayQueue.java:259) at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:323) at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$$Lambda$164/1053864765.run(Unknown Source) at java.lang.Thread.run(Thread.java:750) "AsyncConn Chore Service.Chore.1" #4586 daemon prio=5 os_prio=0 tid=0x00007fc984d2a000 nid=0x2657 waiting on condition [0x00007fc0e490b000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007fc1fc9286b8> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1093) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:750) "ReadOnlyZKClient-cod-1e3xsc06g8mjm-master1.exait3p.z1did1.b0.cloudera.site:2181,cod-1e3xsc06g8mjm-master0.exait3p.z1did1.b0.cloudera.site:2181,cod-1e3xsc06g8mjm-leader0.exait3p.z1did1.b0.cloudera.site:2181@0x5cd5b623" #4581 daemon prio=5 os_prio=0 tid=0x0000558b13d0b800 nid=0x264f waiting on condition [0x00007fc0e4e10000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007fc1fc9288f8> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.DelayQueue.poll(DelayQueue.java:259) at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:323) at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$$Lambda$164/1053864765.run(Unknown Source) at java.lang.Thread.run(Thread.java:750) "AsyncConn Chore Service.Chore.1" #4580 daemon prio=5 os_prio=0 tid=0x0000558b1486a000 nid=0x264e waiting on condition [0x00007fc0e4f11000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007fc1fc92b978> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1093) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:750) "ReadOnlyZKClient-cod-1e3xsc06g8mjm-master1.exait3p.z1did1.b0.cloudera.site:2181,cod-1e3xsc06g8mjm-master0.exait3p.z1did1.b0.cloudera.site:2181,cod-1e3xsc06g8mjm-leader0.exait3p.z1did1.b0.cloudera.site:2181@0x1fd678ff" #4577 daemon prio=5 os_prio=0 tid=0x00007fc97cea5000 nid=0x264b waiting on condition [0x00007fc0e5214000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007fc1fc92bbb8> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.DelayQueue.poll(DelayQueue.java:259) at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:323) at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$$Lambda$164/1053864765.run(Unknown Source) at java.lang.Thread.run(Thread.java:750) "AsyncConn Chore Service.Chore.1" #4576 daemon prio=5 os_prio=0 tid=0x00007fc97c4d5000 nid=0x264a waiting on condition [0x00007fc0e5315000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007fc1fc928af8> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1093) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:750) "ReadOnlyZKClient-cod-1e3xsc06g8mjm-master1.exait3p.z1did1.b0.cloudera.site:2181,cod-1e3xsc06g8mjm-master0.exait3p.z1did1.b0.cloudera.site:2181,cod-1e3xsc06g8mjm-leader0.exait3p.z1did1.b0.cloudera.site:2181@0x4ac7f434" #4564 daemon prio=5 os_prio=0 tid=0x00007fc11e115800 nid=0x263c waiting on condition [0x00007fc0e5e20000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007fc1fbd5d378> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.DelayQueue.poll(DelayQueue.java:259) at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:323) at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$$Lambda$164/1053864765.run(Unknown Source) at java.lang.Thread.run(Thread.java:750) "AsyncConn Chore Service.Chore.1" #4563 daemon prio=5 os_prio=0 tid=0x00007fc11e113800 nid=0x263b waiting on condition [0x00007fc0e5f21000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007fc1fbd5f378> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1093) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:750) "ReadOnlyZKClient-cod-1e3xsc06g8mjm-master1.exait3p.z1did1.b0.cloudera.site:2181,cod-1e3xsc06g8mjm-master0.exait3p.z1did1.b0.cloudera.site:2181,cod-1e3xsc06g8mjm-leader0.exait3p.z1did1.b0.cloudera.site:2181@0x69686825" #4562 daemon prio=5 os_prio=0 tid=0x00007fc11e111800 nid=0x263a waiting on condition [0x00007fc0e6022000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007fc1fbd63378> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.DelayQueue.poll(DelayQueue.java:259) at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:323) at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$$Lambda$164/1053864765.run(Unknown Source) at java.lang.Thread.run(Thread.java:750) "AsyncConn Chore Service.Chore.1" #4561 daemon prio=5 os_prio=0 tid=0x00007fc11e110000 nid=0x2639 waiting on condition [0x00007fc0e6123000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007fc1fbd5d578> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1093) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:750) "ReadOnlyZKClient-cod-1e3xsc06g8mjm-master1.exait3p.z1did1.b0.cloudera.site:2181,cod-1e3xsc06g8mjm-master0.exait3p.z1did1.b0.cloudera.site:2181,cod-1e3xsc06g8mjm-leader0.exait3p.z1did1.b0.cloudera.site:2181@0x30294902" #4542 daemon prio=5 os_prio=0 tid=0x00007fc984af2000 nid=0x261d waiting on condition [0x00007fc0e6c2e000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007fc1fbd5d7b8> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.DelayQueue.poll(DelayQueue.java:259) at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:323) at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$$Lambda$164/1053864765.run(Unknown Source) at java.lang.Thread.run(Thread.java:750) "AsyncConn Chore Service.Chore.1" #4541 daemon prio=5 os_prio=0 tid=0x00007fc984af0800 nid=0x261c waiting on condition [0x00007fc0e6d2f000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007fc1fbd5d9b8> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1093) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:750) "ReadOnlyZKClient-cod-1e3xsc06g8mjm-master1.exait3p.z1did1.b0.cloudera.site:2181,cod-1e3xsc06g8mjm-master0.exait3p.z1did1.b0.cloudera.site:2181,cod-1e3xsc06g8mjm-leader0.exait3p.z1did1.b0.cloudera.site:2181@0x51e80ec9" #4525 daemon prio=5 os_prio=0 tid=0x00007fc11dd07800 nid=0x2603 waiting on condition [0x00007fc0e7638000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007fc1fbd61378> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.DelayQueue.poll(DelayQueue.java:259) at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:323) at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$$Lambda$164/1053864765.run(Unknown Source) at java.lang.Thread.run(Thread.java:750) "AsyncConn Chore Service.Chore.1" #4524 daemon prio=5 os_prio=0 tid=0x00007fc11dd04000 nid=0x2602 waiting on condition [0x00007fc0e7739000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007fc1fbd5dbf8> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1093) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:750) "ReadOnlyZKClient-cod-1e3xsc06g8mjm-master1.exait3p.z1did1.b0.cloudera.site:2181,cod-1e3xsc06g8mjm-master0.exait3p.z1did1.b0.cloudera.site:2181,cod-1e3xsc06g8mjm-leader0.exait3p.z1did1.b0.cloudera.site:2181@0x32fccd3d" #4515 daemon prio=5 os_prio=0 tid=0x0000558b1439a000 nid=0x25f7 waiting on condition [0x00007fc0e7e40000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007fc1fbd61578> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.DelayQueue.poll(DelayQueue.java:259) at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:323) at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$$Lambda$164/1053864765.run(Unknown Source) at java.lang.Thread.run(Thread.java:750) "AsyncConn Chore Service.Chore.1" #4514 daemon prio=5 os_prio=0 tid=0x0000558b14399000 nid=0x25f6 waiting on condition [0x00007fc0ec486000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007fc1fbd5f880> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1093) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:750) "ReadOnlyZKClient-cod-1e3xsc06g8mjm-master1.exait3p.z1did1.b0.cloudera.site:2181,cod-1e3xsc06g8mjm-master0.exait3p.z1did1.b0.cloudera.site:2181,cod-1e3xsc06g8mjm-leader0.exait3p.z1did1.b0.cloudera.site:2181@0x69a95a6a" #4497 daemon prio=5 os_prio=0 tid=0x00007fc97d3de000 nid=0x25dd waiting on condition [0x00007fc0e884a000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007fc1fb11f178> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.DelayQueue.poll(DelayQueue.java:259) at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:323) at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$$Lambda$164/1053864765.run(Unknown Source) at java.lang.Thread.run(Thread.java:750) "AsyncConn Chore Service.Chore.1" #4496 daemon prio=5 os_prio=0 tid=0x00007fc97d3dc800 nid=0x25dc waiting on condition [0x00007fc0eb072000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007fc1fb11f378> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1093) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:750) "ReadOnlyZKClient-cod-1e3xsc06g8mjm-master1.exait3p.z1did1.b0.cloudera.site:2181,cod-1e3xsc06g8mjm-master0.exait3p.z1did1.b0.cloudera.site:2181,cod-1e3xsc06g8mjm-leader0.exait3p.z1did1.b0.cloudera.site:2181@0x181327c3" #4485 daemon prio=5 os_prio=0 tid=0x00007fc9851c2800 nid=0x25ba waiting on condition [0x00007fc0e9153000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007fc1fb11f5b8> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.DelayQueue.poll(DelayQueue.java:259) at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:323) at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$$Lambda$164/1053864765.run(Unknown Source) at java.lang.Thread.run(Thread.java:750) "AsyncConn Chore Service.Chore.1" #4484 daemon prio=5 os_prio=0 tid=0x00007fc9851c1800 nid=0x25b9 waiting on condition [0x00007fc0e9355000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007fc1fb11f7b8> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1093) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:750) "ReadOnlyZKClient-cod-1e3xsc06g8mjm-master1.exait3p.z1did1.b0.cloudera.site:2181,cod-1e3xsc06g8mjm-master0.exait3p.z1did1.b0.cloudera.site:2181,cod-1e3xsc06g8mjm-leader0.exait3p.z1did1.b0.cloudera.site:2181@0x73b3cb4a" #4471 daemon prio=5 os_prio=0 tid=0x0000558b13e6b800 nid=0x25a6 waiting on condition [0x00007fc0e9b5d000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007fc1fb11f9f8> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.DelayQueue.poll(DelayQueue.java:259) at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:323) at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$$Lambda$164/1053864765.run(Unknown Source) at java.lang.Thread.run(Thread.java:750) "AsyncConn Chore Service.Chore.1" #4470 daemon prio=5 os_prio=0 tid=0x0000558b14144800 nid=0x25a5 waiting on condition [0x00007fc0e9c5e000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007fc1fb11fbf8> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1093) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:750) "ReadOnlyZKClient-cod-1e3xsc06g8mjm-master1.exait3p.z1did1.b0.cloudera.site:2181,cod-1e3xsc06g8mjm-master0.exait3p.z1did1.b0.cloudera.site:2181,cod-1e3xsc06g8mjm-leader0.exait3p.z1did1.b0.cloudera.site:2181@0x32e9f13d" #4467 daemon prio=5 os_prio=0 tid=0x00007fc11e423000 nid=0x25a2 waiting on condition [0x00007fc0e9f61000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007fc1fb11fe38> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.DelayQueue.poll(DelayQueue.java:259) at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:323) at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$$Lambda$164/1053864765.run(Unknown Source) at java.lang.Thread.run(Thread.java:750) "AsyncConn Chore Service.Chore.1" #4466 daemon prio=5 os_prio=0 tid=0x00007fc11c72e800 nid=0x25a1 waiting on condition [0x00007fc0ea062000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007fc1fb120038> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1093) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:750) "ReadOnlyZKClient-cod-1e3xsc06g8mjm-master1.exait3p.z1did1.b0.cloudera.site:2181,cod-1e3xsc06g8mjm-master0.exait3p.z1did1.b0.cloudera.site:2181,cod-1e3xsc06g8mjm-leader0.exait3p.z1did1.b0.cloudera.site:2181@0x09075a19" #4445 daemon prio=5 os_prio=0 tid=0x0000558b1457f800 nid=0x2585 waiting on condition [0x00007fc0eae70000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007fc1f9962578> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.DelayQueue.poll(DelayQueue.java:259) at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:323) at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$$Lambda$164/1053864765.run(Unknown Source) at java.lang.Thread.run(Thread.java:750) "AsyncConn Chore Service.Chore.1" #4444 daemon prio=5 os_prio=0 tid=0x0000558b1457f000 nid=0x2584 waiting on condition [0x00007fc0eaf71000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007fc1f9964578> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1093) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:750) "ReadOnlyZKClient-cod-1e3xsc06g8mjm-master1.exait3p.z1did1.b0.cloudera.site:2181,cod-1e3xsc06g8mjm-master0.exait3p.z1did1.b0.cloudera.site:2181,cod-1e3xsc06g8mjm-leader0.exait3p.z1did1.b0.cloudera.site:2181@0x3100860f" #4433 daemon prio=5 os_prio=0 tid=0x00007fc97db4b000 nid=0x2572 waiting on condition [0x00007fc0eb678000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007fc1f99647b8> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.DelayQueue.poll(DelayQueue.java:259) at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:323) at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$$Lambda$164/1053864765.run(Unknown Source) at java.lang.Thread.run(Thread.java:750) "AsyncConn Chore Service.Chore.1" #4432 daemon prio=5 os_prio=0 tid=0x00007fc97dc2a000 nid=0x2571 waiting on condition [0x00007fc0eb779000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007fc1f99649b8> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1093) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:750) "ReadOnlyZKClient-cod-1e3xsc06g8mjm-master1.exait3p.z1did1.b0.cloudera.site:2181,cod-1e3xsc06g8mjm-master0.exait3p.z1did1.b0.cloudera.site:2181,cod-1e3xsc06g8mjm-leader0.exait3p.z1did1.b0.cloudera.site:2181@0x56dd6041" #4422 daemon prio=5 os_prio=0 tid=0x00007fc985528000 nid=0x2566 waiting on condition [0x00007fc0ec183000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007fc1f9960578> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.DelayQueue.poll(DelayQueue.java:259) at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:323) at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$$Lambda$164/1053864765.run(Unknown Source) at java.lang.Thread.run(Thread.java:750) "AsyncConn Chore Service.Chore.1" #4421 daemon prio=5 os_prio=0 tid=0x00007fc985527000 nid=0x2565 waiting on condition [0x00007fc0ec284000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007fc1f9964bf8> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1093) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:750) "ReadOnlyZKClient-cod-1e3xsc06g8mjm-master1.exait3p.z1did1.b0.cloudera.site:2181,cod-1e3xsc06g8mjm-master0.exait3p.z1did1.b0.cloudera.site:2181,cod-1e3xsc06g8mjm-leader0.exait3p.z1did1.b0.cloudera.site:2181@0x44efb9a7" #4409 daemon prio=5 os_prio=0 tid=0x00007fc984549800 nid=0x2557 waiting on condition [0x00007fc0ed89a000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007fc1f9960778> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.DelayQueue.poll(DelayQueue.java:259) at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:323) at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$$Lambda$164/1053864765.run(Unknown Source) at java.lang.Thread.run(Thread.java:750) "AsyncConn Chore Service.Chore.1" #4408 daemon prio=5 os_prio=0 tid=0x00007fc984549000 nid=0x2556 waiting on condition [0x00007fc0eda9c000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007fc1f9964e38> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1093) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:750) "ReadOnlyZKClient-cod-1e3xsc06g8mjm-master1.exait3p.z1did1.b0.cloudera.site:2181,cod-1e3xsc06g8mjm-master0.exait3p.z1did1.b0.cloudera.site:2181,cod-1e3xsc06g8mjm-leader0.exait3p.z1did1.b0.cloudera.site:2181@0x1e37dbf6" #4385 daemon prio=5 os_prio=0 tid=0x00007fc11e438000 nid=0x253c waiting on condition [0x00007fc0ed294000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007fc1f995e578> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.DelayQueue.poll(DelayQueue.java:259) at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:323) at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$$Lambda$164/1053864765.run(Unknown Source) at java.lang.Thread.run(Thread.java:750) "AsyncConn Chore Service.Chore.1" #4384 daemon prio=5 os_prio=0 tid=0x00007fc11e437000 nid=0x253b waiting on condition [0x00007fc0ed395000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007fc1f9960978> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1093) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:750) "ReadOnlyZKClient-cod-1e3xsc06g8mjm-master1.exait3p.z1did1.b0.cloudera.site:2181,cod-1e3xsc06g8mjm-master0.exait3p.z1did1.b0.cloudera.site:2181,cod-1e3xsc06g8mjm-leader0.exait3p.z1did1.b0.cloudera.site:2181@0x54cad721" #4369 daemon prio=5 os_prio=0 tid=0x00007fc98554a000 nid=0x252b waiting on condition [0x00007fc0ee0a2000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007fc1f9965078> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.DelayQueue.poll(DelayQueue.java:259) at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:323) at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$$Lambda$164/1053864765.run(Unknown Source) at java.lang.Thread.run(Thread.java:750) "AsyncConn Chore Service.Chore.1" #4368 daemon prio=5 os_prio=0 tid=0x00007fc985549000 nid=0x252a waiting on condition [0x00007fc0ee1a3000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007fc1f995e778> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1093) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:750) "ReadOnlyZKClient-cod-1e3xsc06g8mjm-master1.exait3p.z1did1.b0.cloudera.site:2181,cod-1e3xsc06g8mjm-master0.exait3p.z1did1.b0.cloudera.site:2181,cod-1e3xsc06g8mjm-leader0.exait3p.z1did1.b0.cloudera.site:2181@0x04326cd0" #4367 daemon prio=5 os_prio=0 tid=0x00007fc11c736800 nid=0x2529 waiting on condition [0x00007fc0ee2a4000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007fc1f9960bb8> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.DelayQueue.poll(DelayQueue.java:259) at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:323) at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$$Lambda$164/1053864765.run(Unknown Source) at java.lang.Thread.run(Thread.java:750) "AsyncConn Chore Service.Chore.1" #4366 daemon prio=5 os_prio=0 tid=0x00007fc11e403800 nid=0x2528 waiting on condition [0x00007fc0ee3a5000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007fc1f995e9b8> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1093) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:750) "ReadOnlyZKClient-cod-1e3xsc06g8mjm-master1.exait3p.z1did1.b0.cloudera.site:2181,cod-1e3xsc06g8mjm-master0.exait3p.z1did1.b0.cloudera.site:2181,cod-1e3xsc06g8mjm-leader0.exait3p.z1did1.b0.cloudera.site:2181@0x159764c9" #4350 daemon prio=5 os_prio=0 tid=0x00007fc11e3ff800 nid=0x2516 waiting on condition [0x00007fc0f01c1000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007fc1f9965278> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.DelayQueue.poll(DelayQueue.java:259) at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:323) at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$$Lambda$164/1053864765.run(Unknown Source) at java.lang.Thread.run(Thread.java:750) "ReadOnlyZKClient-cod-1e3xsc06g8mjm-master1.exait3p.z1did1.b0.cloudera.site:2181,cod-1e3xsc06g8mjm-master0.exait3p.z1did1.b0.cloudera.site:2181,cod-1e3xsc06g8mjm-leader0.exait3p.z1did1.b0.cloudera.site:2181@0x4f1dc748" #4349 daemon prio=5 os_prio=0 tid=0x00007fc11e3ff000 nid=0x2515 waiting on condition [0x00007fc0f06c6000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007fc1f9960db8> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.DelayQueue.poll(DelayQueue.java:259) at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:323) at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$$Lambda$164/1053864765.run(Unknown Source) at java.lang.Thread.run(Thread.java:750) "AsyncConn Chore Service.Chore.1" #4348 daemon prio=5 os_prio=0 tid=0x00007fc11dd55000 nid=0x2514 waiting on condition [0x00007fc0f07c7000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007fc1f995ebf8> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1093) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:750) "AsyncConn Chore Service.Chore.1" #4347 daemon prio=5 os_prio=0 tid=0x00007fc11dd54800 nid=0x2513 waiting on condition [0x00007fc0f0ece000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007fc1f9965478> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1093) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:750) "ReadOnlyZKClient-cod-1e3xsc06g8mjm-master1.exait3p.z1did1.b0.cloudera.site:2181,cod-1e3xsc06g8mjm-master0.exait3p.z1did1.b0.cloudera.site:2181,cod-1e3xsc06g8mjm-leader0.exait3p.z1did1.b0.cloudera.site:2181@0x38d1f359" #4344 daemon prio=5 os_prio=0 tid=0x00007fc97c256000 nid=0x2510 waiting on condition [0x00007fc0f2beb000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007fc1f995ee38> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.DelayQueue.poll(DelayQueue.java:259) at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:323) at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$$Lambda$164/1053864765.run(Unknown Source) at java.lang.Thread.run(Thread.java:750) "AsyncConn Chore Service.Chore.1" #4343 daemon prio=5 os_prio=0 tid=0x00007fc97ca77000 nid=0x250f waiting on condition [0x00007fc0f2ded000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007fc1f99656b8> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1093) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:750) "ReadOnlyZKClient-cod-1e3xsc06g8mjm-master1.exait3p.z1did1.b0.cloudera.site:2181,cod-1e3xsc06g8mjm-master0.exait3p.z1did1.b0.cloudera.site:2181,cod-1e3xsc06g8mjm-leader0.exait3p.z1did1.b0.cloudera.site:2181@0x223af02b" #4333 daemon prio=5 os_prio=0 tid=0x00007fc984120800 nid=0x2504 waiting on condition [0x00007fc0ff0ad000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007fc1f995f038> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.DelayQueue.poll(DelayQueue.java:259) at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:323) at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$$Lambda$164/1053864765.run(Unknown Source) at java.lang.Thread.run(Thread.java:750) "ReadOnlyZKClient-cod-1e3xsc06g8mjm-master1.exait3p.z1did1.b0.cloudera.site:2181,cod-1e3xsc06g8mjm-master0.exait3p.z1did1.b0.cloudera.site:2181,cod-1e3xsc06g8mjm-leader0.exait3p.z1did1.b0.cloudera.site:2181@0x136c36cd" #4332 daemon prio=5 os_prio=0 tid=0x00007fc985394800 nid=0x2503 waiting on condition [0x00007fc0f14d4000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007fc1f9962778> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.DelayQueue.poll(DelayQueue.java:259) at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:323) at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$$Lambda$164/1053864765.run(Unknown Source) at java.lang.Thread.run(Thread.java:750) "AsyncConn Chore Service.Chore.1" #4331 daemon prio=5 os_prio=0 tid=0x00007fc984f28800 nid=0x2502 waiting on condition [0x00007fc0fba77000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007fc1f99658f8> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1093) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:750) "AsyncConn Chore Service.Chore.1" #4330 daemon prio=5 os_prio=0 tid=0x00007fc98509a800 nid=0x2501 waiting on condition [0x00007fc0fc582000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007fc1f9962978> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1093) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:750) "ReadOnlyZKClient-cod-1e3xsc06g8mjm-master1.exait3p.z1did1.b0.cloudera.site:2181,cod-1e3xsc06g8mjm-master0.exait3p.z1did1.b0.cloudera.site:2181,cod-1e3xsc06g8mjm-leader0.exait3p.z1did1.b0.cloudera.site:2181@0x27d5b3df" #4283 daemon prio=5 os_prio=0 tid=0x00007fc984f18000 nid=0x248b waiting on condition [0x00007fc0f2aea000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007fc5b1497798> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.DelayQueue.poll(DelayQueue.java:259) at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:323) at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$$Lambda$164/1053864765.run(Unknown Source) at java.lang.Thread.run(Thread.java:750) "AsyncConn Chore Service.Chore.1" #4282 daemon prio=5 os_prio=0 tid=0x00007fc984f17800 nid=0x248a waiting on condition [0x00007fc0f6726000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007fc5b1497998> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1093) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:750) "ReadOnlyZKClient-cod-1e3xsc06g8mjm-master1.exait3p.z1did1.b0.cloudera.site:2181,cod-1e3xsc06g8mjm-master0.exait3p.z1did1.b0.cloudera.site:2181,cod-1e3xsc06g8mjm-leader0.exait3p.z1did1.b0.cloudera.site:2181@0x2936f488" #4275 daemon prio=5 os_prio=0 tid=0x00007fc97c076800 nid=0x2482 waiting on condition [0x00007fc0f5211000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007fc5b1497bd8> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.DelayQueue.poll(DelayQueue.java:259) at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:323) at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$$Lambda$164/1053864765.run(Unknown Source) at java.lang.Thread.run(Thread.java:750) "AsyncConn Chore Service.Chore.1" #4274 daemon prio=5 os_prio=0 tid=0x00007fc97c075800 nid=0x2481 waiting on condition [0x00007fc0ef5b7000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007fc5b1497dd8> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1093) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:750) "ReadOnlyZKClient-cod-1e3xsc06g8mjm-master1.exait3p.z1did1.b0.cloudera.site:2181,cod-1e3xsc06g8mjm-master0.exait3p.z1did1.b0.cloudera.site:2181,cod-1e3xsc06g8mjm-leader0.exait3p.z1did1.b0.cloudera.site:2181@0x3429dd72" #4265 daemon prio=5 os_prio=0 tid=0x0000558b14605000 nid=0x2477 waiting on condition [0x00007fc0f18d8000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007fc5b1498018> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.DelayQueue.poll(DelayQueue.java:259) at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:323) at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$$Lambda$164/1053864765.run(Unknown Source) at java.lang.Thread.run(Thread.java:750) "ReadOnlyZKClient-cod-1e3xsc06g8mjm-master1.exait3p.z1did1.b0.cloudera.site:2181,cod-1e3xsc06g8mjm-master0.exait3p.z1did1.b0.cloudera.site:2181,cod-1e3xsc06g8mjm-leader0.exait3p.z1did1.b0.cloudera.site:2181@0x6df60a64" #4264 daemon prio=5 os_prio=0 tid=0x00007fc11dc8e800 nid=0x2476 waiting on condition [0x00007fc0f13d3000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007fc5b1498218> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.DelayQueue.poll(DelayQueue.java:259) at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:323) at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$$Lambda$164/1053864765.run(Unknown Source) at java.lang.Thread.run(Thread.java:750) "ReadOnlyZKClient-cod-1e3xsc06g8mjm-master1.exait3p.z1did1.b0.cloudera.site:2181,cod-1e3xsc06g8mjm-master0.exait3p.z1did1.b0.cloudera.site:2181,cod-1e3xsc06g8mjm-leader0.exait3p.z1did1.b0.cloudera.site:2181@0x04657fea" #4263 daemon prio=5 os_prio=0 tid=0x00007fc11dc8e000 nid=0x2475 waiting on condition [0x00007fc0f1ada000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007fc5b1498418> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.DelayQueue.poll(DelayQueue.java:259) at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:323) at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$$Lambda$164/1053864765.run(Unknown Source) at java.lang.Thread.run(Thread.java:750) "AsyncConn Chore Service.Chore.1" #4262 daemon prio=5 os_prio=0 tid=0x0000558b14604000 nid=0x2474 waiting on condition [0x00007fc0f15d5000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007fc5b1498618> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1093) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:750) "AsyncConn Chore Service.Chore.1" #4261 daemon prio=5 os_prio=0 tid=0x00007fc99b99a800 nid=0x2473 waiting on condition [0x00007fc0f36f6000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007fc5b1498858> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1093) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:750) "AsyncConn Chore Service.Chore.1" #4260 daemon prio=5 os_prio=0 tid=0x00007fc11dc60800 nid=0x2472 waiting on condition [0x00007fc0f10d0000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007fc5b1498a98> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1093) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:750) "ReadOnlyZKClient-cod-1e3xsc06g8mjm-master1.exait3p.z1did1.b0.cloudera.site:2181,cod-1e3xsc06g8mjm-master0.exait3p.z1did1.b0.cloudera.site:2181,cod-1e3xsc06g8mjm-leader0.exait3p.z1did1.b0.cloudera.site:2181@0x0f6e7e55" #4255 daemon prio=5 os_prio=0 tid=0x0000558b14602800 nid=0x246c waiting on condition [0x00007fc0f08c8000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007fc5b1498cd8> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.DelayQueue.poll(DelayQueue.java:259) at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:323) at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$$Lambda$164/1053864765.run(Unknown Source) at java.lang.Thread.run(Thread.java:750) "AsyncConn Chore Service.Chore.1" #4254 daemon prio=5 os_prio=0 tid=0x0000558b145ff000 nid=0x246b waiting on condition [0x00007fc0f09c9000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007fc5b1498ed8> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1093) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:750) "ReadOnlyZKClient-cod-1e3xsc06g8mjm-master1.exait3p.z1did1.b0.cloudera.site:2181,cod-1e3xsc06g8mjm-master0.exait3p.z1did1.b0.cloudera.site:2181,cod-1e3xsc06g8mjm-leader0.exait3p.z1did1.b0.cloudera.site:2181@0x38e4469d" #4225 daemon prio=5 os_prio=0 tid=0x00007fc97c3d5000 nid=0x23c9 waiting on condition [0x00007fc0f03c3000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007fc1f72ba778> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.DelayQueue.poll(DelayQueue.java:259) at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:323) at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$$Lambda$164/1053864765.run(Unknown Source) at java.lang.Thread.run(Thread.java:750) "ReadOnlyZKClient-cod-1e3xsc06g8mjm-master1.exait3p.z1did1.b0.cloudera.site:2181,cod-1e3xsc06g8mjm-master0.exait3p.z1did1.b0.cloudera.site:2181,cod-1e3xsc06g8mjm-leader0.exait3p.z1did1.b0.cloudera.site:2181@0x775ac121" #4224 daemon prio=5 os_prio=0 tid=0x0000558b13d16000 nid=0x23c8 waiting on condition [0x00007fc0fad6a000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007fc1f72b8778> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.DelayQueue.poll(DelayQueue.java:259) at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:323) at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$$Lambda$164/1053864765.run(Unknown Source) at java.lang.Thread.run(Thread.java:750) "AsyncConn Chore Service.Chore.1" #4223 daemon prio=5 os_prio=0 tid=0x0000558b145a8800 nid=0x23c7 waiting on condition [0x00007fc0f0ccc000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007fc1f72ba978> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1093) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:750) "AsyncConn Chore Service.Chore.1" #4222 daemon prio=5 os_prio=0 tid=0x00007fc97dc28800 nid=0x23c6 waiting on condition [0x00007fc10ae38000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007fc1f72b6778> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1093) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:750) "ReadOnlyZKClient-cod-1e3xsc06g8mjm-master1.exait3p.z1did1.b0.cloudera.site:2181,cod-1e3xsc06g8mjm-master0.exait3p.z1did1.b0.cloudera.site:2181,cod-1e3xsc06g8mjm-leader0.exait3p.z1did1.b0.cloudera.site:2181@0x42bcf436" #4189 daemon prio=5 os_prio=0 tid=0x00007fc11dd1d000 nid=0x2338 waiting on condition [0x00007fc0f2cec000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007fc5b1499118> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.DelayQueue.poll(DelayQueue.java:259) at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:323) at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$$Lambda$164/1053864765.run(Unknown Source) at java.lang.Thread.run(Thread.java:750) "AsyncConn Chore Service.Chore.1" #4188 daemon prio=5 os_prio=0 tid=0x00007fc11dd1c000 nid=0x2337 waiting on condition [0x00007fc0fd693000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007fc5b1499318> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1093) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:750) "ReadOnlyZKClient-cod-1e3xsc06g8mjm-master1.exait3p.z1did1.b0.cloudera.site:2181,cod-1e3xsc06g8mjm-master0.exait3p.z1did1.b0.cloudera.site:2181,cod-1e3xsc06g8mjm-leader0.exait3p.z1did1.b0.cloudera.site:2181@0x294ebfa5" #4176 daemon prio=5 os_prio=0 tid=0x00007fc11d21f800 nid=0x230c waiting on condition [0x00007fc0f4a09000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007fc5b1499558> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.DelayQueue.poll(DelayQueue.java:259) at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:323) at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$$Lambda$164/1053864765.run(Unknown Source) at java.lang.Thread.run(Thread.java:750) "AsyncConn Chore Service.Chore.1" #4175 daemon prio=5 os_prio=0 tid=0x00007fc11de8b000 nid=0x230b waiting on condition [0x00007fc0f4d0c000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007fc5b1499778> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1093) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:750) "ReadOnlyZKClient-cod-1e3xsc06g8mjm-master1.exait3p.z1did1.b0.cloudera.site:2181,cod-1e3xsc06g8mjm-master0.exait3p.z1did1.b0.cloudera.site:2181,cod-1e3xsc06g8mjm-leader0.exait3p.z1did1.b0.cloudera.site:2181@0x0e883154" #4150 daemon prio=5 os_prio=0 tid=0x00007fc99b9a2000 nid=0x22e5 waiting on condition [0x00007fc0f7e3d000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007fc5b14999b8> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.DelayQueue.poll(DelayQueue.java:259) at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:323) at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$$Lambda$164/1053864765.run(Unknown Source) at java.lang.Thread.run(Thread.java:750) "ReadOnlyZKClient-cod-1e3xsc06g8mjm-master1.exait3p.z1did1.b0.cloudera.site:2181,cod-1e3xsc06g8mjm-master0.exait3p.z1did1.b0.cloudera.site:2181,cod-1e3xsc06g8mjm-leader0.exait3p.z1did1.b0.cloudera.site:2181@0x271722ec" #4149 daemon prio=5 os_prio=0 tid=0x00007fc11ca54800 nid=0x22e4 waiting on condition [0x00007fc0f7f3e000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007fc5b1499bb8> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.DelayQueue.poll(DelayQueue.java:259) at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:323) at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$$Lambda$164/1053864765.run(Unknown Source) at java.lang.Thread.run(Thread.java:750) "AsyncConn Chore Service.Chore.1" #4148 daemon prio=5 os_prio=0 tid=0x00007fc11da62800 nid=0x22e3 waiting on condition [0x00007fc0f8746000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007fc5b1499db8> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1093) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:750) "AsyncConn Chore Service.Chore.1" #4147 daemon prio=5 os_prio=0 tid=0x00007fc99b05a000 nid=0x22e2 waiting on condition [0x00007fc0f8847000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007fc5b1499ff8> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1093) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:750) "ReadOnlyZKClient-cod-1e3xsc06g8mjm-master1.exait3p.z1did1.b0.cloudera.site:2181,cod-1e3xsc06g8mjm-master0.exait3p.z1did1.b0.cloudera.site:2181,cod-1e3xsc06g8mjm-leader0.exait3p.z1did1.b0.cloudera.site:2181@0x72fe09e7" #4137 daemon prio=5 os_prio=0 tid=0x00007fc9847d9000 nid=0x22d1 waiting on condition [0x00007fc0ef6b8000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007fc5b149a238> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.DelayQueue.poll(DelayQueue.java:259) at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:323) at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$$Lambda$164/1053864765.run(Unknown Source) at java.lang.Thread.run(Thread.java:750) "AsyncConn Chore Service.Chore.1" #4136 daemon prio=5 os_prio=0 tid=0x00007fc984941000 nid=0x22d0 waiting on condition [0x00007fc10520c000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007fc5b149a438> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1093) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:750) "ReadOnlyZKClient-cod-1e3xsc06g8mjm-master1.exait3p.z1did1.b0.cloudera.site:2181,cod-1e3xsc06g8mjm-master0.exait3p.z1did1.b0.cloudera.site:2181,cod-1e3xsc06g8mjm-leader0.exait3p.z1did1.b0.cloudera.site:2181@0x6ae61da2" #4121 daemon prio=5 os_prio=0 tid=0x00007fc97c8ce000 nid=0x22be waiting on condition [0x00007fc0f1ddd000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007fc5b149a678> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.DelayQueue.poll(DelayQueue.java:259) at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:323) at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$$Lambda$164/1053864765.run(Unknown Source) at java.lang.Thread.run(Thread.java:750) "ReadOnlyZKClient-cod-1e3xsc06g8mjm-master1.exait3p.z1did1.b0.cloudera.site:2181,cod-1e3xsc06g8mjm-master0.exait3p.z1did1.b0.cloudera.site:2181,cod-1e3xsc06g8mjm-leader0.exait3p.z1did1.b0.cloudera.site:2181@0x0bc30f3f" #4120 daemon prio=5 os_prio=0 tid=0x00007fc984abb800 nid=0x22bd waiting on condition [0x00007fc0f17d7000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007fc5b149a878> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.DelayQueue.poll(DelayQueue.java:259) at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:323) at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$$Lambda$164/1053864765.run(Unknown Source) at java.lang.Thread.run(Thread.java:750) "AsyncConn Chore Service.Chore.1" #4119 daemon prio=5 os_prio=0 tid=0x00007fc97c8cd000 nid=0x22bc waiting on condition [0x00007fc0f28e8000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007fc5b149aa78> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1093) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:750) "AsyncConn Chore Service.Chore.1" #4118 daemon prio=5 os_prio=0 tid=0x00007fc985627800 nid=0x22bb waiting on condition [0x00007fc0f5f1e000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007fc5b149acb8> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1093) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:750) "ReadOnlyZKClient-cod-1e3xsc06g8mjm-master1.exait3p.z1did1.b0.cloudera.site:2181,cod-1e3xsc06g8mjm-master0.exait3p.z1did1.b0.cloudera.site:2181,cod-1e3xsc06g8mjm-leader0.exait3p.z1did1.b0.cloudera.site:2181@0x41ce71b9" #4107 daemon prio=5 os_prio=0 tid=0x00007fc97c252800 nid=0x22a4 waiting on condition [0x00007fc1035f0000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007fc4e7ea8b20> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.DelayQueue.poll(DelayQueue.java:259) at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:323) at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$$Lambda$164/1053864765.run(Unknown Source) at java.lang.Thread.run(Thread.java:750) "AsyncConn Chore Service.Chore.1" #4106 daemon prio=5 os_prio=0 tid=0x00007fc97c578800 nid=0x22a3 waiting on condition [0x00007fc0ed799000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007fc4e7ea8d20> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1093) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:750) "ReadOnlyZKClient-cod-1e3xsc06g8mjm-master1.exait3p.z1did1.b0.cloudera.site:2181,cod-1e3xsc06g8mjm-master0.exait3p.z1did1.b0.cloudera.site:2181,cod-1e3xsc06g8mjm-leader0.exait3p.z1did1.b0.cloudera.site:2181@0x6eccc8b8" #4092 daemon prio=5 os_prio=0 tid=0x00007fc98509e000 nid=0x228e waiting on condition [0x00007fc0f0bcb000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007fc4e7ea8f60> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.DelayQueue.poll(DelayQueue.java:259) at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:323) at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$$Lambda$164/1053864765.run(Unknown Source) at java.lang.Thread.run(Thread.java:750) "ReadOnlyZKClient-cod-1e3xsc06g8mjm-master1.exait3p.z1did1.b0.cloudera.site:2181,cod-1e3xsc06g8mjm-master0.exait3p.z1did1.b0.cloudera.site:2181,cod-1e3xsc06g8mjm-leader0.exait3p.z1did1.b0.cloudera.site:2181@0x4d03b54d" #4091 daemon prio=5 os_prio=0 tid=0x00007fc98509d000 nid=0x228d waiting on condition [0x00007fc1020db000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007fc4e7eacfa0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.DelayQueue.poll(DelayQueue.java:259) at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:323) at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$$Lambda$164/1053864765.run(Unknown Source) at java.lang.Thread.run(Thread.java:750) "AsyncConn Chore Service.Chore.1" #4090 daemon prio=5 os_prio=0 tid=0x00007fc98509c800 nid=0x228c waiting on condition [0x00007fc0fb471000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007fc4e7ea9160> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1093) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:750) "AsyncConn Chore Service.Chore.1" #4089 daemon prio=5 os_prio=0 tid=0x00007fc98509b800 nid=0x228b waiting on condition [0x00007fc1017d2000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007fc4e7ead1a0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1093) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:750) "ReadOnlyZKClient-cod-1e3xsc06g8mjm-master1.exait3p.z1did1.b0.cloudera.site:2181,cod-1e3xsc06g8mjm-master0.exait3p.z1did1.b0.cloudera.site:2181,cod-1e3xsc06g8mjm-leader0.exait3p.z1did1.b0.cloudera.site:2181@0x299e32fd" #4078 daemon prio=5 os_prio=0 tid=0x00007fc9852c6000 nid=0x2276 waiting on condition [0x00007fc0fb370000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007fc4e7eb2978> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.DelayQueue.poll(DelayQueue.java:259) at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:323) at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$$Lambda$164/1053864765.run(Unknown Source) at java.lang.Thread.run(Thread.java:750) "AsyncConn Chore Service.Chore.1" #4077 daemon prio=5 os_prio=0 tid=0x00007fc9852c5800 nid=0x2275 waiting on condition [0x00007fc104c06000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007fc4e7ea93a0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1093) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:750) "ReadOnlyZKClient-cod-1e3xsc06g8mjm-master1.exait3p.z1did1.b0.cloudera.site:2181,cod-1e3xsc06g8mjm-master0.exait3p.z1did1.b0.cloudera.site:2181,cod-1e3xsc06g8mjm-leader0.exait3p.z1did1.b0.cloudera.site:2181@0x00dedc50" #4066 daemon prio=5 os_prio=0 tid=0x00007fc9850f6800 nid=0x2250 waiting on condition [0x00007fc0fb572000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007fc4e7ead3e0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.DelayQueue.poll(DelayQueue.java:259) at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:323) at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$$Lambda$164/1053864765.run(Unknown Source) at java.lang.Thread.run(Thread.java:750) "AsyncConn Chore Service.Chore.1" #4065 daemon prio=5 os_prio=0 tid=0x00007fc9850f6000 nid=0x224f waiting on condition [0x00007fc0fb976000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007fc4e7eb2b78> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1093) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:750) "ReadOnlyZKClient-cod-1e3xsc06g8mjm-master1.exait3p.z1did1.b0.cloudera.site:2181,cod-1e3xsc06g8mjm-master0.exait3p.z1did1.b0.cloudera.site:2181,cod-1e3xsc06g8mjm-leader0.exait3p.z1did1.b0.cloudera.site:2181@0x76018a6d" #3794 daemon prio=5 os_prio=0 tid=0x00007fc11dcda800 nid=0x1f3b waiting on condition [0x00007fc0ee6a8000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007fc1ef287178> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.DelayQueue.poll(DelayQueue.java:259) at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:323) at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$$Lambda$164/1053864765.run(Unknown Source) at java.lang.Thread.run(Thread.java:750) "AsyncConn Chore Service.Chore.1" #3793 daemon prio=5 os_prio=0 tid=0x00007fc11dd02000 nid=0x1f3a waiting on condition [0x00007fc0ee7a9000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007fc1ef289178> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1093) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:750) "ReadOnlyZKClient-cod-1e3xsc06g8mjm-master1.exait3p.z1did1.b0.cloudera.site:2181,cod-1e3xsc06g8mjm-master0.exait3p.z1did1.b0.cloudera.site:2181,cod-1e3xsc06g8mjm-leader0.exait3p.z1did1.b0.cloudera.site:2181@0x7a94f3e6" #3789 daemon prio=5 os_prio=0 tid=0x00007fc9856a6000 nid=0x1f36 waiting on condition [0x00007fc0eeaac000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007fc1ef2893b8> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.DelayQueue.poll(DelayQueue.java:259) at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:323) at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$$Lambda$164/1053864765.run(Unknown Source) at java.lang.Thread.run(Thread.java:750) "AsyncConn Chore Service.Chore.1" #3788 daemon prio=5 os_prio=0 tid=0x00007fc9856a4800 nid=0x1f35 waiting on condition [0x00007fc0eebad000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007fc1ef287378> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1093) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:750) "ReadOnlyZKClient-cod-1e3xsc06g8mjm-master1.exait3p.z1did1.b0.cloudera.site:2181,cod-1e3xsc06g8mjm-master0.exait3p.z1did1.b0.cloudera.site:2181,cod-1e3xsc06g8mjm-leader0.exait3p.z1did1.b0.cloudera.site:2181@0x412b012f" #3780 daemon prio=5 os_prio=0 tid=0x00007fc11dce5000 nid=0x1f2b waiting on condition [0x00007fc0ef2b4000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007fc1ef2895b8> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.DelayQueue.poll(DelayQueue.java:259) at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:323) at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$$Lambda$164/1053864765.run(Unknown Source) at java.lang.Thread.run(Thread.java:750) "AsyncConn Chore Service.Chore.1" #3779 daemon prio=5 os_prio=0 tid=0x00007fc99bce8000 nid=0x1f2a waiting on condition [0x00007fc0ef3b5000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007fc1ef2875b8> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1093) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:750) "ReadOnlyZKClient-cod-1e3xsc06g8mjm-master1.exait3p.z1did1.b0.cloudera.site:2181,cod-1e3xsc06g8mjm-master0.exait3p.z1did1.b0.cloudera.site:2181,cod-1e3xsc06g8mjm-leader0.exait3p.z1did1.b0.cloudera.site:2181@0x0461eb45" #3773 daemon prio=5 os_prio=0 tid=0x00007fc11ddcc000 nid=0x1f24 waiting on condition [0x00007fc0ef9bb000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007fc1ef2897b8> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.DelayQueue.poll(DelayQueue.java:259) at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:323) at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$$Lambda$164/1053864765.run(Unknown Source) at java.lang.Thread.run(Thread.java:750) "AsyncConn Chore Service.Chore.1" #3772 daemon prio=5 os_prio=0 tid=0x00007fc99b187000 nid=0x1f23 waiting on condition [0x00007fc0f02c2000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007fc1ef2899b8> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1093) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:750) "ReadOnlyZKClient-cod-1e3xsc06g8mjm-master1.exait3p.z1did1.b0.cloudera.site:2181,cod-1e3xsc06g8mjm-master0.exait3p.z1did1.b0.cloudera.site:2181,cod-1e3xsc06g8mjm-leader0.exait3p.z1did1.b0.cloudera.site:2181@0x6149807a" #3764 daemon prio=5 os_prio=0 tid=0x00007fc99b992000 nid=0x1f1b waiting on condition [0x00007fc0f1ede000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007fc1ef2877f8> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.DelayQueue.poll(DelayQueue.java:259) at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:323) at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$$Lambda$164/1053864765.run(Unknown Source) at java.lang.Thread.run(Thread.java:750) "AsyncConn Chore Service.Chore.1" #3763 daemon prio=5 os_prio=0 tid=0x00007fc99ae72000 nid=0x1f1a waiting on condition [0x00007fc0f601f000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007fc1ef289bf8> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1093) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:750) "ReadOnlyZKClient-cod-1e3xsc06g8mjm-master1.exait3p.z1did1.b0.cloudera.site:2181,cod-1e3xsc06g8mjm-master0.exait3p.z1did1.b0.cloudera.site:2181,cod-1e3xsc06g8mjm-leader0.exait3p.z1did1.b0.cloudera.site:2181@0x5b0f68d3" #3758 daemon prio=5 os_prio=0 tid=0x00007fc984b85000 nid=0x1f15 waiting on condition [0x00007fc0f5615000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007fc1ef2879f8> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.DelayQueue.poll(DelayQueue.java:259) at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:323) at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$$Lambda$164/1053864765.run(Unknown Source) at java.lang.Thread.run(Thread.java:750) "AsyncConn Chore Service.Chore.1" #3757 daemon prio=5 os_prio=0 tid=0x00007fc984579000 nid=0x1f14 waiting on condition [0x00007fc0f7837000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007fc1ef289e38> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1093) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:750) "ReadOnlyZKClient-cod-1e3xsc06g8mjm-master1.exait3p.z1did1.b0.cloudera.site:2181,cod-1e3xsc06g8mjm-master0.exait3p.z1did1.b0.cloudera.site:2181,cod-1e3xsc06g8mjm-leader0.exait3p.z1did1.b0.cloudera.site:2181@0x6fe9374a" #3723 daemon prio=5 os_prio=0 tid=0x00007fc97c526800 nid=0x1ee1 waiting on condition [0x00007fc109828000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007fc1eec04780> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.DelayQueue.poll(DelayQueue.java:259) at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:323) at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$$Lambda$164/1053864765.run(Unknown Source) at java.lang.Thread.run(Thread.java:750) "ReadOnlyZKClient-cod-1e3xsc06g8mjm-master1.exait3p.z1did1.b0.cloudera.site:2181,cod-1e3xsc06g8mjm-master0.exait3p.z1did1.b0.cloudera.site:2181,cod-1e3xsc06g8mjm-leader0.exait3p.z1did1.b0.cloudera.site:2181@0x2fcc1c42" #3722 daemon prio=5 os_prio=0 tid=0x00007fc97caf7000 nid=0x1ee0 waiting on condition [0x00007fc0ff9b6000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007fc1eec00780> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.DelayQueue.poll(DelayQueue.java:259) at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:323) at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$$Lambda$164/1053864765.run(Unknown Source) at java.lang.Thread.run(Thread.java:750) "AsyncConn Chore Service.Chore.1" #3721 daemon prio=5 os_prio=0 tid=0x00007fc97cd79800 nid=0x1edf waiting on condition [0x00007fc0f7736000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007fc1eec06780> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1093) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:750) "AsyncConn Chore Service.Chore.1" #3720 daemon prio=5 os_prio=0 tid=0x00007fc97c2da000 nid=0x1ede waiting on condition [0x00007fc102fea000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007fc1eec02780> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1093) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:750) "ReadOnlyZKClient-cod-1e3xsc06g8mjm-master1.exait3p.z1did1.b0.cloudera.site:2181,cod-1e3xsc06g8mjm-master0.exait3p.z1did1.b0.cloudera.site:2181,cod-1e3xsc06g8mjm-leader0.exait3p.z1did1.b0.cloudera.site:2181@0x24f118b3" #3657 daemon prio=5 os_prio=0 tid=0x00007fc984f96800 nid=0x1e20 waiting on condition [0x00007fc0f16d6000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007fc1ed98ed78> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.DelayQueue.poll(DelayQueue.java:259) at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:323) at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$$Lambda$164/1053864765.run(Unknown Source) at java.lang.Thread.run(Thread.java:750) "AsyncConn Chore Service.Chore.1" #3656 daemon prio=5 os_prio=0 tid=0x00007fc9855c6800 nid=0x1e1f waiting on condition [0x00007fc0f34f4000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007fc1ed996d78> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1093) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:750) "ReadOnlyZKClient-cod-1e3xsc06g8mjm-master1.exait3p.z1did1.b0.cloudera.site:2181,cod-1e3xsc06g8mjm-master0.exait3p.z1did1.b0.cloudera.site:2181,cod-1e3xsc06g8mjm-leader0.exait3p.z1did1.b0.cloudera.site:2181@0x2fd9b783" #3651 daemon prio=5 os_prio=0 tid=0x0000558b140c2800 nid=0x1e1a waiting on condition [0x00007fc0f4100000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007fc1ed994d78> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.DelayQueue.poll(DelayQueue.java:259) at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:323) at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$$Lambda$164/1053864765.run(Unknown Source) at java.lang.Thread.run(Thread.java:750) "ReadOnlyZKClient-cod-1e3xsc06g8mjm-master1.exait3p.z1did1.b0.cloudera.site:2181,cod-1e3xsc06g8mjm-master0.exait3p.z1did1.b0.cloudera.site:2181,cod-1e3xsc06g8mjm-leader0.exait3p.z1did1.b0.cloudera.site:2181@0x57364377" #3650 daemon prio=5 os_prio=0 tid=0x00007fc99b184800 nid=0x1e19 waiting on condition [0x00007fc0f4201000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007fc1ed99ae98> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.DelayQueue.poll(DelayQueue.java:259) at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:323) at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$$Lambda$164/1053864765.run(Unknown Source) at java.lang.Thread.run(Thread.java:750) "AsyncConn Chore Service.Chore.1" #3649 daemon prio=5 os_prio=0 tid=0x0000558b1444c000 nid=0x1e18 waiting on condition [0x00007fc0f4302000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007fc1ed998d78> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1093) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:750) "AsyncConn Chore Service.Chore.1" #3648 daemon prio=5 os_prio=0 tid=0x00007fc11d03a800 nid=0x1e17 waiting on condition [0x00007fc0f4908000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007fc1ed98ef78> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1093) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:750) "ReadOnlyZKClient-cod-1e3xsc06g8mjm-master1.exait3p.z1did1.b0.cloudera.site:2181,cod-1e3xsc06g8mjm-master0.exait3p.z1did1.b0.cloudera.site:2181,cod-1e3xsc06g8mjm-leader0.exait3p.z1did1.b0.cloudera.site:2181@0x501d6788" #3627 daemon prio=5 os_prio=0 tid=0x00007fc984f73800 nid=0x1dfd waiting on condition [0x00007fc0fa05d000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007fc5d3808378> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.DelayQueue.poll(DelayQueue.java:259) at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:323) at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$$Lambda$164/1053864765.run(Unknown Source) at java.lang.Thread.run(Thread.java:750) "AsyncConn Chore Service.Chore.1" #3626 daemon prio=5 os_prio=0 tid=0x00007fc985496000 nid=0x1dfc waiting on condition [0x00007fc0fa15e000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007fc5d3808578> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1093) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:750) "ReadOnlyZKClient-cod-1e3xsc06g8mjm-master1.exait3p.z1did1.b0.cloudera.site:2181,cod-1e3xsc06g8mjm-master0.exait3p.z1did1.b0.cloudera.site:2181,cod-1e3xsc06g8mjm-leader0.exait3p.z1did1.b0.cloudera.site:2181@0x6e4b9c0d" #3609 daemon prio=5 os_prio=0 tid=0x00007fc984df5800 nid=0x1de9 waiting on condition [0x00007fc100fca000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007fc5d38087b8> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.DelayQueue.poll(DelayQueue.java:259) at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:323) at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$$Lambda$164/1053864765.run(Unknown Source) at java.lang.Thread.run(Thread.java:750) "ReadOnlyZKClient-cod-1e3xsc06g8mjm-master1.exait3p.z1did1.b0.cloudera.site:2181,cod-1e3xsc06g8mjm-master0.exait3p.z1did1.b0.cloudera.site:2181,cod-1e3xsc06g8mjm-leader0.exait3p.z1did1.b0.cloudera.site:2181@0x454fb758" #3608 daemon prio=5 os_prio=0 tid=0x0000558b1476c000 nid=0x1de8 waiting on condition [0x00007fc0fdb98000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007fc5d38089b8> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.DelayQueue.poll(DelayQueue.java:259) at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:323) at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$$Lambda$164/1053864765.run(Unknown Source) at java.lang.Thread.run(Thread.java:750) "AsyncConn Chore Service.Chore.1" #3607 daemon prio=5 os_prio=0 tid=0x00007fc98478f800 nid=0x1de7 waiting on condition [0x00007fc1000bb000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007fc5d3808bb8> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1093) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:750) "AsyncConn Chore Service.Chore.1" #3606 daemon prio=5 os_prio=0 tid=0x0000558b1476b000 nid=0x1de6 waiting on condition [0x00007fc0fd08d000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007fc5d3808df8> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1093) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:750) "ReadOnlyZKClient-cod-1e3xsc06g8mjm-master1.exait3p.z1did1.b0.cloudera.site:2181,cod-1e3xsc06g8mjm-master0.exait3p.z1did1.b0.cloudera.site:2181,cod-1e3xsc06g8mjm-leader0.exait3p.z1did1.b0.cloudera.site:2181@0x4aed71d2" #3601 daemon prio=5 os_prio=0 tid=0x00007fc984508800 nid=0x1de0 waiting on condition [0x00007fc10500a000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007fc5d3809038> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.DelayQueue.poll(DelayQueue.java:259) at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:323) at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$$Lambda$164/1053864765.run(Unknown Source) at java.lang.Thread.run(Thread.java:750) "AsyncConn Chore Service.Chore.1" #3600 daemon prio=5 os_prio=0 tid=0x00007fc984cf2000 nid=0x1ddf waiting on condition [0x00007fc103bf4000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007fc5d3809238> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1093) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:750) "ReadOnlyZKClient-cod-1e3xsc06g8mjm-master1.exait3p.z1did1.b0.cloudera.site:2181,cod-1e3xsc06g8mjm-master0.exait3p.z1did1.b0.cloudera.site:2181,cod-1e3xsc06g8mjm-leader0.exait3p.z1did1.b0.cloudera.site:2181@0x3c3e08d7" #3596 daemon prio=5 os_prio=0 tid=0x00007fc11dd1b000 nid=0x1dda waiting on condition [0x00007fc0fae6b000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007fc5d3809478> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.DelayQueue.poll(DelayQueue.java:259) at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:323) at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$$Lambda$164/1053864765.run(Unknown Source) at java.lang.Thread.run(Thread.java:750) "AsyncConn Chore Service.Chore.1" #3595 daemon prio=5 os_prio=0 tid=0x00007fc11dbf4000 nid=0x1dd9 waiting on condition [0x00007fc0fe19e000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007fc5d3809678> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1093) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:750) "ReadOnlyZKClient-cod-1e3xsc06g8mjm-master1.exait3p.z1did1.b0.cloudera.site:2181,cod-1e3xsc06g8mjm-master0.exait3p.z1did1.b0.cloudera.site:2181,cod-1e3xsc06g8mjm-leader0.exait3p.z1did1.b0.cloudera.site:2181@0x6eeb6e2a" #3530 daemon prio=5 os_prio=0 tid=0x00007fc97ce59000 nid=0x1d3f waiting on condition [0x00007fc0f5716000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007fc681289178> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.DelayQueue.poll(DelayQueue.java:259) at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:323) at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$$Lambda$164/1053864765.run(Unknown Source) at java.lang.Thread.run(Thread.java:750) "AsyncConn Chore Service.Chore.1" #3529 daemon prio=5 os_prio=0 tid=0x00007fc97ce57800 nid=0x1d3e waiting on condition [0x00007fc0f29e9000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007fc681287178> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1093) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:750) "ReadOnlyZKClient-cod-1e3xsc06g8mjm-master1.exait3p.z1did1.b0.cloudera.site:2181,cod-1e3xsc06g8mjm-master0.exait3p.z1did1.b0.cloudera.site:2181,cod-1e3xsc06g8mjm-leader0.exait3p.z1did1.b0.cloudera.site:2181@0x5e694a47" #3525 daemon prio=5 os_prio=0 tid=0x00007fc99b0a5800 nid=0x1d39 waiting on condition [0x00007fc0f3dfd000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007fc681289378> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.DelayQueue.poll(DelayQueue.java:259) at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:323) at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$$Lambda$164/1053864765.run(Unknown Source) at java.lang.Thread.run(Thread.java:750) "AsyncConn Chore Service.Chore.1" #3524 daemon prio=5 os_prio=0 tid=0x00007fc11ce51000 nid=0x1d38 waiting on condition [0x00007fc0f27e7000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007fc6812873b8> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1093) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:750) "ReadOnlyZKClient-cod-1e3xsc06g8mjm-master1.exait3p.z1did1.b0.cloudera.site:2181,cod-1e3xsc06g8mjm-master0.exait3p.z1did1.b0.cloudera.site:2181,cod-1e3xsc06g8mjm-leader0.exait3p.z1did1.b0.cloudera.site:2181@0x0a7ad756" #3521 daemon prio=5 os_prio=0 tid=0x0000558b14468000 nid=0x1d35 waiting on condition [0x00007fc0f1fdf000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007fc681289578> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.DelayQueue.poll(DelayQueue.java:259) at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:323) at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$$Lambda$164/1053864765.run(Unknown Source) at java.lang.Thread.run(Thread.java:750) "AsyncConn Chore Service.Chore.1" #3518 daemon prio=5 os_prio=0 tid=0x0000558b1414b000 nid=0x1d32 waiting on condition [0x00007fc0f20e0000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007fc6812875f8> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1093) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:750) "ReadOnlyZKClient-cod-1e3xsc06g8mjm-master1.exait3p.z1did1.b0.cloudera.site:2181,cod-1e3xsc06g8mjm-master0.exait3p.z1did1.b0.cloudera.site:2181,cod-1e3xsc06g8mjm-leader0.exait3p.z1did1.b0.cloudera.site:2181@0x2716838f" #3517 daemon prio=5 os_prio=0 tid=0x00007fc984bcc000 nid=0x1d31 waiting on condition [0x00007fc0f1bdb000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007fc681289778> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.DelayQueue.poll(DelayQueue.java:259) at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:323) at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$$Lambda$164/1053864765.run(Unknown Source) at java.lang.Thread.run(Thread.java:750) "AsyncConn Chore Service.Chore.1" #3516 daemon prio=5 os_prio=0 tid=0x00007fc9847e8000 nid=0x1d30 waiting on condition [0x00007fc0f19d9000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007fc681287838> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1093) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:750) "ReadOnlyZKClient-cod-1e3xsc06g8mjm-master1.exait3p.z1did1.b0.cloudera.site:2181,cod-1e3xsc06g8mjm-master0.exait3p.z1did1.b0.cloudera.site:2181,cod-1e3xsc06g8mjm-leader0.exait3p.z1did1.b0.cloudera.site:2181@0x1a0958e7" #3489 daemon prio=5 os_prio=0 tid=0x00007fc11e12a000 nid=0x1c72 waiting on condition [0x00007fc0f5d1c000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007fc1eb55e578> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.DelayQueue.poll(DelayQueue.java:259) at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:323) at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$$Lambda$164/1053864765.run(Unknown Source) at java.lang.Thread.run(Thread.java:750) "AsyncConn Chore Service.Chore.1" #3488 daemon prio=5 os_prio=0 tid=0x00007fc11df0f800 nid=0x1c71 waiting on condition [0x00007fc1019d4000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007fc1eb562578> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1093) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:750) "ReadOnlyZKClient-cod-1e3xsc06g8mjm-master1.exait3p.z1did1.b0.cloudera.site:2181,cod-1e3xsc06g8mjm-master0.exait3p.z1did1.b0.cloudera.site:2181,cod-1e3xsc06g8mjm-leader0.exait3p.z1did1.b0.cloudera.site:2181@0x1c809cd7" #3485 daemon prio=5 os_prio=0 tid=0x00007fc11e195800 nid=0x1c6e waiting on condition [0x00007fc0ffab7000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007fc1eb55e778> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.DelayQueue.poll(DelayQueue.java:259) at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:323) at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$$Lambda$164/1053864765.run(Unknown Source) at java.lang.Thread.run(Thread.java:750) "AsyncConn Chore Service.Chore.1" #3484 daemon prio=5 os_prio=0 tid=0x00007fc11dd6b000 nid=0x1c6d waiting on condition [0x00007fc0fd491000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007fc1eb560578> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1093) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:750) "ReadOnlyZKClient-cod-1e3xsc06g8mjm-master1.exait3p.z1did1.b0.cloudera.site:2181,cod-1e3xsc06g8mjm-master0.exait3p.z1did1.b0.cloudera.site:2181,cod-1e3xsc06g8mjm-leader0.exait3p.z1did1.b0.cloudera.site:2181@0x1eed1e72" #3471 daemon prio=5 os_prio=0 tid=0x00007fc99b834800 nid=0x1c3a waiting on condition [0x00007fc100bc6000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007fc1eb5627b8> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.DelayQueue.poll(DelayQueue.java:259) at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:323) at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$$Lambda$164/1053864765.run(Unknown Source) at java.lang.Thread.run(Thread.java:750) "AsyncConn Chore Service.Chore.1" #3470 daemon prio=5 os_prio=0 tid=0x00007fc11e128800 nid=0x1c39 waiting on condition [0x00007fc0fedaa000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007fc1eb55e978> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1093) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:750) "ReadOnlyZKClient-cod-1e3xsc06g8mjm-master1.exait3p.z1did1.b0.cloudera.site:2181,cod-1e3xsc06g8mjm-master0.exait3p.z1did1.b0.cloudera.site:2181,cod-1e3xsc06g8mjm-leader0.exait3p.z1did1.b0.cloudera.site:2181@0x72806ad6" #3390 daemon prio=5 os_prio=0 tid=0x00007fc11df6b000 nid=0x1b82 waiting on condition [0x00007fc0f803f000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007fc1ea8efb78> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.DelayQueue.poll(DelayQueue.java:259) at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:323) at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$$Lambda$164/1053864765.run(Unknown Source) at java.lang.Thread.run(Thread.java:750) "AsyncConn Chore Service.Chore.1" #3389 daemon prio=5 os_prio=0 tid=0x00007fc11df6b800 nid=0x1b81 waiting on condition [0x00007fc0f33f3000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007fc1ea8ebbb8> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1093) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:750) "ReadOnlyZKClient-cod-1e3xsc06g8mjm-master1.exait3p.z1did1.b0.cloudera.site:2181,cod-1e3xsc06g8mjm-master0.exait3p.z1did1.b0.cloudera.site:2181,cod-1e3xsc06g8mjm-leader0.exait3p.z1did1.b0.cloudera.site:2181@0x20627ef1" #3379 daemon prio=5 os_prio=0 tid=0x00007fc984714800 nid=0x1b6f waiting on condition [0x00007fc0f35f5000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007fc1e90d8f78> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.DelayQueue.poll(DelayQueue.java:259) at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:323) at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$$Lambda$164/1053864765.run(Unknown Source) at java.lang.Thread.run(Thread.java:750) "AsyncConn Chore Service.Chore.1" #3378 daemon prio=5 os_prio=0 tid=0x00007fc984714000 nid=0x1b6e waiting on condition [0x00007fc0f9c59000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007fc1e90dcf78> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1093) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:750) "ReadOnlyZKClient-cod-1e3xsc06g8mjm-master1.exait3p.z1did1.b0.cloudera.site:2181,cod-1e3xsc06g8mjm-master0.exait3p.z1did1.b0.cloudera.site:2181,cod-1e3xsc06g8mjm-leader0.exait3p.z1did1.b0.cloudera.site:2181@0x55ce70ef" #3365 daemon prio=5 os_prio=0 tid=0x0000558b145ec800 nid=0x1b5d waiting on condition [0x00007fc103cf5000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007fc1e90dd1b8> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.DelayQueue.poll(DelayQueue.java:259) at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:323) at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$$Lambda$164/1053864765.run(Unknown Source) at java.lang.Thread.run(Thread.java:750) "AsyncConn Chore Service.Chore.1" #3364 daemon prio=5 os_prio=0 tid=0x0000558b145eb800 nid=0x1b5c waiting on condition [0x00007fc1044ff000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007fc1e90dd3b8> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1093) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:750) "ReadOnlyZKClient-cod-1e3xsc06g8mjm-master1.exait3p.z1did1.b0.cloudera.site:2181,cod-1e3xsc06g8mjm-master0.exait3p.z1did1.b0.cloudera.site:2181,cod-1e3xsc06g8mjm-leader0.exait3p.z1did1.b0.cloudera.site:2181@0x728f1c97" #3353 daemon prio=5 os_prio=0 tid=0x00007fc984a57800 nid=0x1b50 waiting on condition [0x00007fc0f3efe000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007fc1e90daf78> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.DelayQueue.poll(DelayQueue.java:259) at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:323) at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$$Lambda$164/1053864765.run(Unknown Source) at java.lang.Thread.run(Thread.java:750) "AsyncConn Chore Service.Chore.1" #3352 daemon prio=5 os_prio=0 tid=0x00007fc984a56800 nid=0x1b4f waiting on condition [0x00007fc0f4403000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007fc1e90dd5f8> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1093) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:750) "ReadOnlyZKClient-cod-1e3xsc06g8mjm-master1.exait3p.z1did1.b0.cloudera.site:2181,cod-1e3xsc06g8mjm-master0.exait3p.z1did1.b0.cloudera.site:2181,cod-1e3xsc06g8mjm-leader0.exait3p.z1did1.b0.cloudera.site:2181@0x7ec32fd5" #3334 daemon prio=5 os_prio=0 tid=0x00007fc9847b9800 nid=0x1b32 waiting on condition [0x00007fc0f7635000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007fc1e90db178> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.DelayQueue.poll(DelayQueue.java:259) at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:323) at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$$Lambda$164/1053864765.run(Unknown Source) at java.lang.Thread.run(Thread.java:750) "AsyncConn Chore Service.Chore.1" #3333 daemon prio=5 os_prio=0 tid=0x00007fc984188800 nid=0x1b31 waiting on condition [0x00007fc1026e1000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007fc1e90dd838> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1093) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:750) "ReadOnlyZKClient-cod-1e3xsc06g8mjm-master1.exait3p.z1did1.b0.cloudera.site:2181,cod-1e3xsc06g8mjm-master0.exait3p.z1did1.b0.cloudera.site:2181,cod-1e3xsc06g8mjm-leader0.exait3p.z1did1.b0.cloudera.site:2181@0x7564cc9c" #3330 daemon prio=5 os_prio=0 tid=0x00007fc11cc82000 nid=0x1b2e waiting on condition [0x00007fc103ef7000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007fc1e90db378> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.DelayQueue.poll(DelayQueue.java:259) at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:323) at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$$Lambda$164/1053864765.run(Unknown Source) at java.lang.Thread.run(Thread.java:750) "AsyncConn Chore Service.Chore.1" #3329 daemon prio=5 os_prio=0 tid=0x00007fc11c6f1000 nid=0x1b2d waiting on condition [0x00007fc0f8b4a000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007fc1e90dda78> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1093) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:750) "ReadOnlyZKClient-cod-1e3xsc06g8mjm-master1.exait3p.z1did1.b0.cloudera.site:2181,cod-1e3xsc06g8mjm-master0.exait3p.z1did1.b0.cloudera.site:2181,cod-1e3xsc06g8mjm-leader0.exait3p.z1did1.b0.cloudera.site:2181@0x38a617c1" #3317 daemon prio=5 os_prio=0 tid=0x00007fc97cace000 nid=0x1b08 waiting on condition [0x00007fc0f4504000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007fc1e90ddcb8> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.DelayQueue.poll(DelayQueue.java:259) at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:323) at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$$Lambda$164/1053864765.run(Unknown Source) at java.lang.Thread.run(Thread.java:750) "AsyncConn Chore Service.Chore.1" #3316 daemon prio=5 os_prio=0 tid=0x00007fc97d9e3800 nid=0x1b07 waiting on condition [0x00007fc0fa764000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007fc1e90db578> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1093) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:750) "ReadOnlyZKClient-cod-1e3xsc06g8mjm-master1.exait3p.z1did1.b0.cloudera.site:2181,cod-1e3xsc06g8mjm-master0.exait3p.z1did1.b0.cloudera.site:2181,cod-1e3xsc06g8mjm-leader0.exait3p.z1did1.b0.cloudera.site:2181@0x0b9e6e01" #3300 daemon prio=5 os_prio=0 tid=0x00007fc9849a3800 nid=0x1af0 waiting on condition [0x00007fc0fc784000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007fc1e90ddeb8> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.DelayQueue.poll(DelayQueue.java:259) at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:323) at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$$Lambda$164/1053864765.run(Unknown Source) at java.lang.Thread.run(Thread.java:750) "AsyncConn Chore Service.Chore.1" #3299 daemon prio=5 os_prio=0 tid=0x00007fc984717800 nid=0x1aef waiting on condition [0x00007fc0fd996000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007fc1e90db7b8> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1093) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:750) "ReadOnlyZKClient-cod-1e3xsc06g8mjm-master1.exait3p.z1did1.b0.cloudera.site:2181,cod-1e3xsc06g8mjm-master0.exait3p.z1did1.b0.cloudera.site:2181,cod-1e3xsc06g8mjm-leader0.exait3p.z1did1.b0.cloudera.site:2181@0x4c094389" #3292 daemon prio=5 os_prio=0 tid=0x00007fc11dd64800 nid=0x1ae7 waiting on condition [0x00007fc0fe8a5000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007fc1e90de0b8> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.DelayQueue.poll(DelayQueue.java:259) at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:323) at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$$Lambda$164/1053864765.run(Unknown Source) at java.lang.Thread.run(Thread.java:750) "AsyncConn Chore Service.Chore.1" #3291 daemon prio=5 os_prio=0 tid=0x00007fc11dc66800 nid=0x1ae6 waiting on condition [0x00007fc0ff6b3000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007fc1e90db9f8> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1093) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:750) "ReadOnlyZKClient-cod-1e3xsc06g8mjm-master1.exait3p.z1did1.b0.cloudera.site:2181,cod-1e3xsc06g8mjm-master0.exait3p.z1did1.b0.cloudera.site:2181,cod-1e3xsc06g8mjm-leader0.exait3p.z1did1.b0.cloudera.site:2181@0x6050474e" #3280 daemon prio=5 os_prio=0 tid=0x0000558b142c9000 nid=0x1ad6 waiting on condition [0x00007fc0f9552000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007fc1e90d6f78> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.DelayQueue.poll(DelayQueue.java:259) at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:323) at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$$Lambda$164/1053864765.run(Unknown Source) at java.lang.Thread.run(Thread.java:750) "AsyncConn Chore Service.Chore.1" #3279 daemon prio=5 os_prio=0 tid=0x0000558b14672000 nid=0x1ad5 waiting on condition [0x00007fc0f5514000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007fc1e90dbc38> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1093) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:750) "ReadOnlyZKClient-cod-1e3xsc06g8mjm-master1.exait3p.z1did1.b0.cloudera.site:2181,cod-1e3xsc06g8mjm-master0.exait3p.z1did1.b0.cloudera.site:2181,cod-1e3xsc06g8mjm-leader0.exait3p.z1did1.b0.cloudera.site:2181@0x56ce01bd" #3260 daemon prio=5 os_prio=0 tid=0x00007fc11df48000 nid=0x1abe waiting on condition [0x00007fc109424000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007fc1e90de2b8> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.DelayQueue.poll(DelayQueue.java:259) at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:323) at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$$Lambda$164/1053864765.run(Unknown Source) at java.lang.Thread.run(Thread.java:750) "AsyncConn Chore Service.Chore.1" #3259 daemon prio=5 os_prio=0 tid=0x00007fc99ac79800 nid=0x1abd waiting on condition [0x00007fc0f4807000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007fc1e90d7178> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1093) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:750) "ReadOnlyZKClient-cod-1e3xsc06g8mjm-master1.exait3p.z1did1.b0.cloudera.site:2181,cod-1e3xsc06g8mjm-master0.exait3p.z1did1.b0.cloudera.site:2181,cod-1e3xsc06g8mjm-leader0.exait3p.z1did1.b0.cloudera.site:2181@0x641e56ab" #3256 daemon prio=5 os_prio=0 tid=0x00007fc11dbf1800 nid=0x1aba waiting on condition [0x00007fc0f4605000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007fc1e90dbe78> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.DelayQueue.poll(DelayQueue.java:259) at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:323) at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$$Lambda$164/1053864765.run(Unknown Source) at java.lang.Thread.run(Thread.java:750) "AsyncConn Chore Service.Chore.1" #3255 daemon prio=5 os_prio=0 tid=0x00007fc11df6a000 nid=0x1ab9 waiting on condition [0x00007fc0f4706000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007fc1e90d73b8> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1093) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:750) "ReadOnlyZKClient-cod-1e3xsc06g8mjm-master1.exait3p.z1did1.b0.cloudera.site:2181,cod-1e3xsc06g8mjm-master0.exait3p.z1did1.b0.cloudera.site:2181,cod-1e3xsc06g8mjm-leader0.exait3p.z1did1.b0.cloudera.site:2181@0x18672956" #3240 daemon prio=5 os_prio=0 tid=0x00007fc11d91f800 nid=0x1aa0 waiting on condition [0x00007fc0f4c0b000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007fc1e90de4b8> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.DelayQueue.poll(DelayQueue.java:259) at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:323) at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$$Lambda$164/1053864765.run(Unknown Source) at java.lang.Thread.run(Thread.java:750) "AsyncConn Chore Service.Chore.1" #3239 daemon prio=5 os_prio=0 tid=0x00007fc11d268000 nid=0x1a9f waiting on condition [0x00007fc0f4e0d000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007fc1e90dc078> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1093) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:750) "ReadOnlyZKClient-cod-1e3xsc06g8mjm-master1.exait3p.z1did1.b0.cloudera.site:2181,cod-1e3xsc06g8mjm-master0.exait3p.z1did1.b0.cloudera.site:2181,cod-1e3xsc06g8mjm-leader0.exait3p.z1did1.b0.cloudera.site:2181@0x69bfc513" #3218 daemon prio=5 os_prio=0 tid=0x00007fc984575800 nid=0x1a09 waiting on condition [0x00007fc0f31f1000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007fc1e800e178> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.DelayQueue.poll(DelayQueue.java:259) at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:323) at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$$Lambda$164/1053864765.run(Unknown Source) at java.lang.Thread.run(Thread.java:750) "AsyncConn Chore Service.Chore.1" #3217 daemon prio=5 os_prio=0 tid=0x00007fc984803800 nid=0x1a08 waiting on condition [0x00007fc0f32f2000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007fc1e80161c0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1093) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:750) "ReadOnlyZKClient-cod-1e3xsc06g8mjm-master1.exait3p.z1did1.b0.cloudera.site:2181,cod-1e3xsc06g8mjm-master0.exait3p.z1did1.b0.cloudera.site:2181,cod-1e3xsc06g8mjm-leader0.exait3p.z1did1.b0.cloudera.site:2181@0x70d914f8" #3206 daemon prio=5 os_prio=0 tid=0x00007fc97c487000 nid=0x19f9 waiting on condition [0x00007fc0f3afa000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007fc1e800c178> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.DelayQueue.poll(DelayQueue.java:259) at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:323) at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$$Lambda$164/1053864765.run(Unknown Source) at java.lang.Thread.run(Thread.java:750) "AsyncConn Chore Service.Chore.1" #3205 daemon prio=5 os_prio=0 tid=0x00007fc97c486800 nid=0x19f8 waiting on condition [0x00007fc0f3bfb000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007fc1e800e378> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1093) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:750) "ReadOnlyZKClient-cod-1e3xsc06g8mjm-master1.exait3p.z1did1.b0.cloudera.site:2181,cod-1e3xsc06g8mjm-master0.exait3p.z1did1.b0.cloudera.site:2181,cod-1e3xsc06g8mjm-leader0.exait3p.z1did1.b0.cloudera.site:2181@0x1c2e63ec" #3190 daemon prio=5 os_prio=0 tid=0x00007fc97c280800 nid=0x19df waiting on condition [0x00007fc0faa67000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007fc1e8016400> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.DelayQueue.poll(DelayQueue.java:259) at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:323) at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$$Lambda$164/1053864765.run(Unknown Source) at java.lang.Thread.run(Thread.java:750) "AsyncConn Chore Service.Chore.1" #3189 daemon prio=5 os_prio=0 tid=0x00007fc97c551800 nid=0x19de waiting on condition [0x00007fc0f8f4e000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007fc1e800c378> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1093) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:750) "ReadOnlyZKClient-cod-1e3xsc06g8mjm-master1.exait3p.z1did1.b0.cloudera.site:2181,cod-1e3xsc06g8mjm-master0.exait3p.z1did1.b0.cloudera.site:2181,cod-1e3xsc06g8mjm-leader0.exait3p.z1did1.b0.cloudera.site:2181@0x2c5d2aaf" #3177 daemon prio=5 os_prio=0 tid=0x00007fc984bcd000 nid=0x19cc waiting on condition [0x00007fc0f8d4c000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007fc1e800e5b8> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.DelayQueue.poll(DelayQueue.java:259) at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:323) at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$$Lambda$164/1053864765.run(Unknown Source) at java.lang.Thread.run(Thread.java:750) "AsyncConn Chore Service.Chore.1" #3176 daemon prio=5 os_prio=0 tid=0x00007fc9849e2800 nid=0x19cb waiting on condition [0x00007fc0f8e4d000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007fc1e8016600> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1093) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:750) "ReadOnlyZKClient-cod-1e3xsc06g8mjm-master1.exait3p.z1did1.b0.cloudera.site:2181,cod-1e3xsc06g8mjm-master0.exait3p.z1did1.b0.cloudera.site:2181,cod-1e3xsc06g8mjm-leader0.exait3p.z1did1.b0.cloudera.site:2181@0x24bf3782" #3162 daemon prio=5 os_prio=0 tid=0x0000558b140c2000 nid=0x19a2 waiting on condition [0x00007fc0feaa7000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007fc1e800c5b8> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.DelayQueue.poll(DelayQueue.java:259) at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:323) at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$$Lambda$164/1053864765.run(Unknown Source) at java.lang.Thread.run(Thread.java:750) "AsyncConn Chore Service.Chore.1" #3161 daemon prio=5 os_prio=0 tid=0x0000558b140c1000 nid=0x19a1 waiting on condition [0x00007fc0f6a29000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007fc1e800e7b8> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1093) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:750) "ReadOnlyZKClient-cod-1e3xsc06g8mjm-master1.exait3p.z1did1.b0.cloudera.site:2181,cod-1e3xsc06g8mjm-master0.exait3p.z1did1.b0.cloudera.site:2181,cod-1e3xsc06g8mjm-leader0.exait3p.z1did1.b0.cloudera.site:2181@0x6b7545fc" #3150 daemon prio=5 os_prio=0 tid=0x00007fc99bea8000 nid=0x1994 waiting on condition [0x00007fc0fd794000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007fc1e8010178> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.DelayQueue.poll(DelayQueue.java:259) at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:323) at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$$Lambda$164/1053864765.run(Unknown Source) at java.lang.Thread.run(Thread.java:750) "AsyncConn Chore Service.Chore.1" #3149 daemon prio=5 os_prio=0 tid=0x00007fc99bcfa000 nid=0x1993 waiting on condition [0x00007fc0ff7b4000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007fc1e800e9f8> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1093) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:750) "ReadOnlyZKClient-cod-1e3xsc06g8mjm-master1.exait3p.z1did1.b0.cloudera.site:2181,cod-1e3xsc06g8mjm-master0.exait3p.z1did1.b0.cloudera.site:2181,cod-1e3xsc06g8mjm-leader0.exait3p.z1did1.b0.cloudera.site:2181@0x2f9f1e98" #3137 daemon prio=5 os_prio=0 tid=0x00007fc9847e9000 nid=0x1985 waiting on condition [0x00007fc1007c2000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007fc1e8010378> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.DelayQueue.poll(DelayQueue.java:259) at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:323) at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$$Lambda$164/1053864765.run(Unknown Source) at java.lang.Thread.run(Thread.java:750) "AsyncConn Chore Service.Chore.1" #3136 daemon prio=5 os_prio=0 tid=0x00007fc984793000 nid=0x1984 waiting on condition [0x00007fc0fb26f000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007fc1e800ec38> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1093) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:750) "ReadOnlyZKClient-cod-1e3xsc06g8mjm-master1.exait3p.z1did1.b0.cloudera.site:2181,cod-1e3xsc06g8mjm-master0.exait3p.z1did1.b0.cloudera.site:2181,cod-1e3xsc06g8mjm-leader0.exait3p.z1did1.b0.cloudera.site:2181@0x151cface" #3127 daemon prio=5 os_prio=0 tid=0x0000558b142d7800 nid=0x197a waiting on condition [0x00007fc0ff5b2000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007fc1e8010578> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.DelayQueue.poll(DelayQueue.java:259) at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:323) at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$$Lambda$164/1053864765.run(Unknown Source) at java.lang.Thread.run(Thread.java:750) "AsyncConn Chore Service.Chore.1" #3126 daemon prio=5 os_prio=0 tid=0x0000558b14675000 nid=0x1979 waiting on condition [0x00007fc0fd895000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007fc1e800ee78> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1093) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:750) "ReadOnlyZKClient-cod-1e3xsc06g8mjm-master1.exait3p.z1did1.b0.cloudera.site:2181,cod-1e3xsc06g8mjm-master0.exait3p.z1did1.b0.cloudera.site:2181,cod-1e3xsc06g8mjm-leader0.exait3p.z1did1.b0.cloudera.site:2181@0x7b0021c8" #3112 daemon prio=5 os_prio=0 tid=0x00007fc984ede800 nid=0x195e waiting on condition [0x00007fc10510b000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007fc1e71b8f78> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.DelayQueue.poll(DelayQueue.java:259) at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:323) at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$$Lambda$164/1053864765.run(Unknown Source) at java.lang.Thread.run(Thread.java:750) "AsyncConn Chore Service.Chore.1" #3110 daemon prio=5 os_prio=0 tid=0x00007fc984abd000 nid=0x195c waiting on condition [0x00007fc0ff8b5000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007fc1e71b9178> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1093) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:750) "ReadOnlyZKClient-cod-1e3xsc06g8mjm-master1.exait3p.z1did1.b0.cloudera.site:2181,cod-1e3xsc06g8mjm-master0.exait3p.z1did1.b0.cloudera.site:2181,cod-1e3xsc06g8mjm-leader0.exait3p.z1did1.b0.cloudera.site:2181@0x40ac3725" #3101 daemon prio=5 os_prio=0 tid=0x00007fc984a38800 nid=0x1949 waiting on condition [0x00007fc0f9f5c000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007fc1e71b93b8> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.DelayQueue.poll(DelayQueue.java:259) at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:323) at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$$Lambda$164/1053864765.run(Unknown Source) at java.lang.Thread.run(Thread.java:750) "AsyncConn Chore Service.Chore.1" #3100 daemon prio=5 os_prio=0 tid=0x00007fc9845a2000 nid=0x1948 waiting on condition [0x00007fc104903000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007fc1e71b95b8> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1093) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:750) "ReadOnlyZKClient-cod-1e3xsc06g8mjm-master1.exait3p.z1did1.b0.cloudera.site:2181,cod-1e3xsc06g8mjm-master0.exait3p.z1did1.b0.cloudera.site:2181,cod-1e3xsc06g8mjm-leader0.exait3p.z1did1.b0.cloudera.site:2181@0x4af1e16f" #3083 daemon prio=5 os_prio=0 tid=0x00007fc11dbf3000 nid=0x1922 waiting on condition [0x00007fc0f4f0e000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007fc1e71b97f8> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.DelayQueue.poll(DelayQueue.java:259) at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:323) at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$$Lambda$164/1053864765.run(Unknown Source) at java.lang.Thread.run(Thread.java:750) "AsyncConn Chore Service.Chore.1" #3082 daemon prio=5 os_prio=0 tid=0x00007fc11dbf2000 nid=0x1921 waiting on condition [0x00007fc0f500f000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007fc1e71b99f8> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1093) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:750) "ReadOnlyZKClient-cod-1e3xsc06g8mjm-master1.exait3p.z1did1.b0.cloudera.site:2181,cod-1e3xsc06g8mjm-master0.exait3p.z1did1.b0.cloudera.site:2181,cod-1e3xsc06g8mjm-leader0.exait3p.z1did1.b0.cloudera.site:2181@0x170c06ec" #3060 daemon prio=5 os_prio=0 tid=0x00007fc11df0e000 nid=0x1901 waiting on condition [0x00007fc0f904f000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007fc1e71b9c38> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.DelayQueue.poll(DelayQueue.java:259) at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:323) at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$$Lambda$164/1053864765.run(Unknown Source) at java.lang.Thread.run(Thread.java:750) "AsyncConn Chore Service.Chore.1" #3059 daemon prio=5 os_prio=0 tid=0x00007fc11ca9b800 nid=0x1900 waiting on condition [0x00007fc0f9150000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007fc1e71b9e38> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1093) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:750) "ReadOnlyZKClient-cod-1e3xsc06g8mjm-master1.exait3p.z1did1.b0.cloudera.site:2181,cod-1e3xsc06g8mjm-master0.exait3p.z1did1.b0.cloudera.site:2181,cod-1e3xsc06g8mjm-leader0.exait3p.z1did1.b0.cloudera.site:2181@0x40005fe9" #3056 daemon prio=5 os_prio=0 tid=0x00007fc984c05000 nid=0x18fd waiting on condition [0x00007fc0f9956000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007fc1e71ba078> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.DelayQueue.poll(DelayQueue.java:259) at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:323) at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$$Lambda$164/1053864765.run(Unknown Source) at java.lang.Thread.run(Thread.java:750) "AsyncConn Chore Service.Chore.1" #3055 daemon prio=5 os_prio=0 tid=0x00007fc9850c6800 nid=0x18fc waiting on condition [0x00007fc0f9e5b000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007fc1e71ba278> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1093) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:750) "ReadOnlyZKClient-cod-1e3xsc06g8mjm-master1.exait3p.z1did1.b0.cloudera.site:2181,cod-1e3xsc06g8mjm-master0.exait3p.z1did1.b0.cloudera.site:2181,cod-1e3xsc06g8mjm-leader0.exait3p.z1did1.b0.cloudera.site:2181@0x57ab4bad" #3046 daemon prio=5 os_prio=0 tid=0x00007fc97c81e000 nid=0x18f2 waiting on condition [0x00007fc0fc885000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007fc1e71ba4b8> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.DelayQueue.poll(DelayQueue.java:259) at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:323) at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$$Lambda$164/1053864765.run(Unknown Source) at java.lang.Thread.run(Thread.java:750) "AsyncConn Chore Service.Chore.1" #3045 daemon prio=5 os_prio=0 tid=0x00007fc97d60f800 nid=0x18f1 waiting on condition [0x00007fc0fc986000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007fc1e71ba6b8> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1093) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:750) "ReadOnlyZKClient-cod-1e3xsc06g8mjm-master1.exait3p.z1did1.b0.cloudera.site:2181,cod-1e3xsc06g8mjm-master0.exait3p.z1did1.b0.cloudera.site:2181,cod-1e3xsc06g8mjm-leader0.exait3p.z1did1.b0.cloudera.site:2181@0x54016f6f" #3025 daemon prio=5 os_prio=0 tid=0x00007fc99b98e800 nid=0x18c1 waiting on condition [0x00007fc0fa461000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007fc67f996218> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.DelayQueue.poll(DelayQueue.java:259) at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:323) at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$$Lambda$164/1053864765.run(Unknown Source) at java.lang.Thread.run(Thread.java:750) "ReadOnlyZKClient-cod-1e3xsc06g8mjm-master1.exait3p.z1did1.b0.cloudera.site:2181,cod-1e3xsc06g8mjm-master0.exait3p.z1did1.b0.cloudera.site:2181,cod-1e3xsc06g8mjm-leader0.exait3p.z1did1.b0.cloudera.site:2181@0x6030236a" #3024 daemon prio=5 os_prio=0 tid=0x0000558b148c3800 nid=0x18c0 waiting on condition [0x00007fc103af3000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007fc67f9a0338> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.DelayQueue.poll(DelayQueue.java:259) at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:323) at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$$Lambda$164/1053864765.run(Unknown Source) at java.lang.Thread.run(Thread.java:750) "AsyncConn Chore Service.Chore.1" #3023 daemon prio=5 os_prio=0 tid=0x00007fc99b98e000 nid=0x18bf waiting on condition [0x00007fc0f6b2a000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007fc67f996418> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1093) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:750) "AsyncConn Chore Service.Chore.1" #3022 daemon prio=5 os_prio=0 tid=0x0000558b13e3b800 nid=0x18be waiting on condition [0x00007fc0f6625000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007fc67f9a0538> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1093) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:750) "ReadOnlyZKClient-cod-1e3xsc06g8mjm-master1.exait3p.z1did1.b0.cloudera.site:2181,cod-1e3xsc06g8mjm-master0.exait3p.z1did1.b0.cloudera.site:2181,cod-1e3xsc06g8mjm-leader0.exait3p.z1did1.b0.cloudera.site:2181@0x177be775" #3006 daemon prio=5 os_prio=0 tid=0x00007fc11dcb4800 nid=0x18a6 waiting on condition [0x00007fc0f6221000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007fc67f996658> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.DelayQueue.poll(DelayQueue.java:259) at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:323) at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$$Lambda$164/1053864765.run(Unknown Source) at java.lang.Thread.run(Thread.java:750) "AsyncConn Chore Service.Chore.1" #3005 daemon prio=5 os_prio=0 tid=0x00007fc11dcb3000 nid=0x18a5 waiting on condition [0x00007fc0f6322000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007fc67f9a0778> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1093) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:750) "ReadOnlyZKClient-cod-1e3xsc06g8mjm-master1.exait3p.z1did1.b0.cloudera.site:2181,cod-1e3xsc06g8mjm-master0.exait3p.z1did1.b0.cloudera.site:2181,cod-1e3xsc06g8mjm-leader0.exait3p.z1did1.b0.cloudera.site:2181@0x6c2263ad" #2991 daemon prio=5 os_prio=0 tid=0x00007fc11dcb8000 nid=0x1896 waiting on condition [0x00007fc0f5a19000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007fc67f996858> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.DelayQueue.poll(DelayQueue.java:259) at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:323) at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$$Lambda$164/1053864765.run(Unknown Source) at java.lang.Thread.run(Thread.java:750) "AsyncConn Chore Service.Chore.1" #2990 daemon prio=5 os_prio=0 tid=0x00007fc11de42000 nid=0x1895 waiting on condition [0x00007fc1043fe000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007fc67f996a58> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1093) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:750) "ReadOnlyZKClient-cod-1e3xsc06g8mjm-master1.exait3p.z1did1.b0.cloudera.site:2181,cod-1e3xsc06g8mjm-master0.exait3p.z1did1.b0.cloudera.site:2181,cod-1e3xsc06g8mjm-leader0.exait3p.z1did1.b0.cloudera.site:2181@0x3c4d5c50" #2987 daemon prio=5 os_prio=0 tid=0x00007fc984bd0800 nid=0x1892 waiting on condition [0x00007fc0f5918000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007fc67f9a09b8> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.DelayQueue.poll(DelayQueue.java:259) at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:323) at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$$Lambda$164/1053864765.run(Unknown Source) at java.lang.Thread.run(Thread.java:750) "AsyncConn Chore Service.Chore.1" #2986 daemon prio=5 os_prio=0 tid=0x00007fc984dc1800 nid=0x1891 waiting on condition [0x00007fc0f5817000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007fc67f996c98> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1093) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:750) "ReadOnlyZKClient-cod-1e3xsc06g8mjm-master1.exait3p.z1did1.b0.cloudera.site:2181,cod-1e3xsc06g8mjm-master0.exait3p.z1did1.b0.cloudera.site:2181,cod-1e3xsc06g8mjm-leader0.exait3p.z1did1.b0.cloudera.site:2181@0x4ab916f2" #2961 daemon prio=5 os_prio=0 tid=0x00007fc97c553000 nid=0x1839 waiting on condition [0x00007fc0f8544000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007fc67f9a0bb8> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.DelayQueue.poll(DelayQueue.java:259) at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:323) at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$$Lambda$164/1053864765.run(Unknown Source) at java.lang.Thread.run(Thread.java:750) "AsyncConn Chore Service.Chore.1" #2960 daemon prio=5 os_prio=0 tid=0x00007fc97c556000 nid=0x1838 waiting on condition [0x00007fc0f8948000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007fc67f996ed8> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1093) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:750) "ReadOnlyZKClient-cod-1e3xsc06g8mjm-master1.exait3p.z1did1.b0.cloudera.site:2181,cod-1e3xsc06g8mjm-master0.exait3p.z1did1.b0.cloudera.site:2181,cod-1e3xsc06g8mjm-leader0.exait3p.z1did1.b0.cloudera.site:2181@0x2ad0bb78" #2957 daemon prio=5 os_prio=0 tid=0x00007fc97c608000 nid=0x1835 waiting on condition [0x00007fc0f7d3c000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007fc67f9a0db8> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.DelayQueue.poll(DelayQueue.java:259) at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:323) at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$$Lambda$164/1053864765.run(Unknown Source) at java.lang.Thread.run(Thread.java:750) "AsyncConn Chore Service.Chore.1" #2956 daemon prio=5 os_prio=0 tid=0x00007fc97d4ec800 nid=0x1834 waiting on condition [0x00007fc0f7b3a000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007fc67f998720> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1093) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:750) "ReadOnlyZKClient-cod-1e3xsc06g8mjm-master1.exait3p.z1did1.b0.cloudera.site:2181,cod-1e3xsc06g8mjm-master0.exait3p.z1did1.b0.cloudera.site:2181,cod-1e3xsc06g8mjm-leader0.exait3p.z1did1.b0.cloudera.site:2181@0x3c3c5040" #2941 daemon prio=5 os_prio=0 tid=0x00007fc98478d000 nid=0x181d waiting on condition [0x00007fc0fa966000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007fc1e4f8c378> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.DelayQueue.poll(DelayQueue.java:259) at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:323) at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$$Lambda$164/1053864765.run(Unknown Source) at java.lang.Thread.run(Thread.java:750) "AsyncConn Chore Service.Chore.1" #2940 daemon prio=5 os_prio=0 tid=0x00007fc98478c800 nid=0x181c waiting on condition [0x00007fc0fac69000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007fc1e4f8c578> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1093) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:750) "ReadOnlyZKClient-cod-1e3xsc06g8mjm-master1.exait3p.z1did1.b0.cloudera.site:2181,cod-1e3xsc06g8mjm-master0.exait3p.z1did1.b0.cloudera.site:2181,cod-1e3xsc06g8mjm-leader0.exait3p.z1did1.b0.cloudera.site:2181@0x5cd3fbaf" #2931 daemon prio=5 os_prio=0 tid=0x0000558b148c6000 nid=0x1811 waiting on condition [0x00007fc100ac5000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007fc1e4f8c7b8> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.DelayQueue.poll(DelayQueue.java:259) at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:323) at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$$Lambda$164/1053864765.run(Unknown Source) at java.lang.Thread.run(Thread.java:750) "AsyncConn Chore Service.Chore.1" #2930 daemon prio=5 os_prio=0 tid=0x0000558b13d3c000 nid=0x1810 waiting on condition [0x00007fc1011cc000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007fc1e4f8c9b8> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1093) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:750) "ReadOnlyZKClient-cod-1e3xsc06g8mjm-master1.exait3p.z1did1.b0.cloudera.site:2181,cod-1e3xsc06g8mjm-master0.exait3p.z1did1.b0.cloudera.site:2181,cod-1e3xsc06g8mjm-leader0.exait3p.z1did1.b0.cloudera.site:2181@0x7841e600" #2915 daemon prio=5 os_prio=0 tid=0x00007fc99b39e000 nid=0x17fe waiting on condition [0x00007fc1006c1000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007fc1e4f8cbf8> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.DelayQueue.poll(DelayQueue.java:259) at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:323) at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$$Lambda$164/1053864765.run(Unknown Source) at java.lang.Thread.run(Thread.java:750) "AsyncConn Chore Service.Chore.1" #2914 daemon prio=5 os_prio=0 tid=0x00007fc99abe3800 nid=0x17fd waiting on condition [0x00007fc0ff4b1000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007fc1e4f8cdf8> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1093) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:750) "ReadOnlyZKClient-cod-1e3xsc06g8mjm-master1.exait3p.z1did1.b0.cloudera.site:2181,cod-1e3xsc06g8mjm-master0.exait3p.z1did1.b0.cloudera.site:2181,cod-1e3xsc06g8mjm-leader0.exait3p.z1did1.b0.cloudera.site:2181@0x70a8a4ff" #2898 daemon prio=5 os_prio=0 tid=0x0000558b13d39800 nid=0x17eb waiting on condition [0x00007fc0fdc99000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007fc1e4f8d038> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.DelayQueue.poll(DelayQueue.java:259) at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:323) at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$$Lambda$164/1053864765.run(Unknown Source) at java.lang.Thread.run(Thread.java:750) "AsyncConn Chore Service.Chore.1" #2897 daemon prio=5 os_prio=0 tid=0x0000558b142d5800 nid=0x17ea waiting on condition [0x00007fc0fdd9a000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007fc1e4f8d238> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1093) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:750) "ReadOnlyZKClient-cod-1e3xsc06g8mjm-master1.exait3p.z1did1.b0.cloudera.site:2181,cod-1e3xsc06g8mjm-master0.exait3p.z1did1.b0.cloudera.site:2181,cod-1e3xsc06g8mjm-leader0.exait3p.z1did1.b0.cloudera.site:2181@0x2a4da4b1" #2896 daemon prio=5 os_prio=0 tid=0x00007fc9843b4000 nid=0x17e9 waiting on condition [0x00007fc0fdf9c000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007fc1e4f8d478> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.DelayQueue.poll(DelayQueue.java:259) at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:323) at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$$Lambda$164/1053864765.run(Unknown Source) at java.lang.Thread.run(Thread.java:750) "AsyncConn Chore Service.Chore.1" #2895 daemon prio=5 os_prio=0 tid=0x00007fc9847bb800 nid=0x17e8 waiting on condition [0x00007fc0fde9b000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007fc1e4f8d678> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1093) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:750) "ReadOnlyZKClient-cod-1e3xsc06g8mjm-master1.exait3p.z1did1.b0.cloudera.site:2181,cod-1e3xsc06g8mjm-master0.exait3p.z1did1.b0.cloudera.site:2181,cod-1e3xsc06g8mjm-leader0.exait3p.z1did1.b0.cloudera.site:2181@0x1e3e846b" #2880 daemon prio=5 os_prio=0 tid=0x00007fc984a5b000 nid=0x17ca waiting on condition [0x00007fc1203fa000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007fc5b149b778> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.DelayQueue.poll(DelayQueue.java:259) at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:323) at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$$Lambda$164/1053864765.run(Unknown Source) at java.lang.Thread.run(Thread.java:750) "AsyncConn Chore Service.Chore.1" #2879 daemon prio=5 os_prio=0 tid=0x00007fc984a5a000 nid=0x17c9 waiting on condition [0x00007fc103df6000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007fc5b149b978> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1093) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:750) "ReadOnlyZKClient-cod-1e3xsc06g8mjm-master1.exait3p.z1did1.b0.cloudera.site:2181,cod-1e3xsc06g8mjm-master0.exait3p.z1did1.b0.cloudera.site:2181,cod-1e3xsc06g8mjm-leader0.exait3p.z1did1.b0.cloudera.site:2181@0x59670f72" #2870 daemon prio=5 os_prio=0 tid=0x00007fc984a56000 nid=0x17bf waiting on condition [0x00007fc104a04000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007fc5b149bbb8> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.DelayQueue.poll(DelayQueue.java:259) at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:323) at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$$Lambda$164/1053864765.run(Unknown Source) at java.lang.Thread.run(Thread.java:750) "AsyncConn Chore Service.Chore.1" #2869 daemon prio=5 os_prio=0 tid=0x00007fc984a55000 nid=0x17be waiting on condition [0x00007fc104b05000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007fc5b149bdb8> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1093) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:750) "ReadOnlyZKClient-cod-1e3xsc06g8mjm-master1.exait3p.z1did1.b0.cloudera.site:2181,cod-1e3xsc06g8mjm-master0.exait3p.z1did1.b0.cloudera.site:2181,cod-1e3xsc06g8mjm-leader0.exait3p.z1did1.b0.cloudera.site:2181@0x35e96aee" #2802 daemon prio=5 os_prio=0 tid=0x0000558b147b0000 nid=0x16e9 waiting on condition [0x00007fc0f7231000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007fc1e337ef78> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.DelayQueue.poll(DelayQueue.java:259) at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:323) at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$$Lambda$164/1053864765.run(Unknown Source) at java.lang.Thread.run(Thread.java:750) "ReadOnlyZKClient-cod-1e3xsc06g8mjm-master1.exait3p.z1did1.b0.cloudera.site:2181,cod-1e3xsc06g8mjm-master0.exait3p.z1did1.b0.cloudera.site:2181,cod-1e3xsc06g8mjm-leader0.exait3p.z1did1.b0.cloudera.site:2181@0x2aea955a" #2801 daemon prio=5 os_prio=0 tid=0x00007fc11db55800 nid=0x16e8 waiting on condition [0x00007fc0f7332000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007fc1e337cf78> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.DelayQueue.poll(DelayQueue.java:259) at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:323) at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$$Lambda$164/1053864765.run(Unknown Source) at java.lang.Thread.run(Thread.java:750) "AsyncConn Chore Service.Chore.1" #2800 daemon prio=5 os_prio=0 tid=0x0000558b147ae000 nid=0x16e7 waiting on condition [0x00007fc0f7433000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007fc1e337af78> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1093) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:750) "AsyncConn Chore Service.Chore.1" #2799 daemon prio=5 os_prio=0 tid=0x00007fc11db54800 nid=0x16e6 waiting on condition [0x00007fc0f7534000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007fc1e337f178> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1093) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:750) "ReadOnlyZKClient-cod-1e3xsc06g8mjm-master1.exait3p.z1did1.b0.cloudera.site:2181,cod-1e3xsc06g8mjm-master0.exait3p.z1did1.b0.cloudera.site:2181,cod-1e3xsc06g8mjm-leader0.exait3p.z1did1.b0.cloudera.site:2181@0x0019a238" #2783 daemon prio=5 os_prio=0 tid=0x00007fc97d4eb800 nid=0x16c0 waiting on condition [0x00007fc0f8140000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007fc1e337d178> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.DelayQueue.poll(DelayQueue.java:259) at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:323) at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$$Lambda$164/1053864765.run(Unknown Source) at java.lang.Thread.run(Thread.java:750) "ReadOnlyZKClient-cod-1e3xsc06g8mjm-master1.exait3p.z1did1.b0.cloudera.site:2181,cod-1e3xsc06g8mjm-master0.exait3p.z1did1.b0.cloudera.site:2181,cod-1e3xsc06g8mjm-leader0.exait3p.z1did1.b0.cloudera.site:2181@0x375ada16" #2784 daemon prio=5 os_prio=0 tid=0x00007fc11ce25000 nid=0x16bf waiting on condition [0x00007fc0f8241000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007fc1e337b1b8> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.DelayQueue.poll(DelayQueue.java:259) at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:323) at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$$Lambda$164/1053864765.run(Unknown Source) at java.lang.Thread.run(Thread.java:750) "AsyncConn Chore Service.Chore.1" #2782 daemon prio=5 os_prio=0 tid=0x00007fc99b9a0000 nid=0x16be waiting on condition [0x00007fc0f8342000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007fc1e3380f78> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1093) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:750) "AsyncConn Chore Service.Chore.1" #2781 daemon prio=5 os_prio=0 tid=0x00007fc97c54f800 nid=0x16bd waiting on condition [0x00007fc0f8443000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007fc1e337d378> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1093) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:750) "ReadOnlyZKClient-cod-1e3xsc06g8mjm-master1.exait3p.z1did1.b0.cloudera.site:2181,cod-1e3xsc06g8mjm-master0.exait3p.z1did1.b0.cloudera.site:2181,cod-1e3xsc06g8mjm-leader0.exait3p.z1did1.b0.cloudera.site:2181@0x2c18dd5a" #2759 daemon prio=5 os_prio=0 tid=0x00007fc11db34800 nid=0x16a2 waiting on condition [0x00007fc0fb673000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007fc6f42d0480> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.DelayQueue.poll(DelayQueue.java:259) at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:323) at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$$Lambda$164/1053864765.run(Unknown Source) at java.lang.Thread.run(Thread.java:750) "AsyncConn Chore Service.Chore.1" #2758 daemon prio=5 os_prio=0 tid=0x00007fc11db33800 nid=0x16a1 waiting on condition [0x00007fc0fb774000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007fc6f42d0680> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1093) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:750) "ReadOnlyZKClient-cod-1e3xsc06g8mjm-master1.exait3p.z1did1.b0.cloudera.site:2181,cod-1e3xsc06g8mjm-master0.exait3p.z1did1.b0.cloudera.site:2181,cod-1e3xsc06g8mjm-leader0.exait3p.z1did1.b0.cloudera.site:2181@0x32c02d2c" #2755 daemon prio=5 os_prio=0 tid=0x0000558b14a23000 nid=0x169e waiting on condition [0x00007fc0fbe7b000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007fc6f42ce678> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.DelayQueue.poll(DelayQueue.java:259) at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:323) at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$$Lambda$164/1053864765.run(Unknown Source) at java.lang.Thread.run(Thread.java:750) "AsyncConn Chore Service.Chore.1" #2754 daemon prio=5 os_prio=0 tid=0x0000558b14a21800 nid=0x169d waiting on condition [0x00007fc0fc481000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007fc6f42d08c0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1093) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:750) "ReadOnlyZKClient-cod-1e3xsc06g8mjm-master1.exait3p.z1did1.b0.cloudera.site:2181,cod-1e3xsc06g8mjm-master0.exait3p.z1did1.b0.cloudera.site:2181,cod-1e3xsc06g8mjm-leader0.exait3p.z1did1.b0.cloudera.site:2181@0x472ff661" #2749 daemon prio=5 os_prio=0 tid=0x00007fc11db31000 nid=0x1698 waiting on condition [0x00007fc0fca87000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007fc6f42ce878> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.DelayQueue.poll(DelayQueue.java:259) at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:323) at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$$Lambda$164/1053864765.run(Unknown Source) at java.lang.Thread.run(Thread.java:750) "AsyncConn Chore Service.Chore.1" #2748 daemon prio=5 os_prio=0 tid=0x00007fc11db30800 nid=0x1697 waiting on condition [0x00007fc0fce8b000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007fc6f42ca650> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1093) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:750) "ReadOnlyZKClient-cod-1e3xsc06g8mjm-master1.exait3p.z1did1.b0.cloudera.site:2181,cod-1e3xsc06g8mjm-master0.exait3p.z1did1.b0.cloudera.site:2181,cod-1e3xsc06g8mjm-leader0.exait3p.z1did1.b0.cloudera.site:2181@0x215e4839" #2745 daemon prio=5 os_prio=0 tid=0x00007fc97c554000 nid=0x1694 waiting on condition [0x00007fc0fd18e000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007fc6f42d0b00> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.DelayQueue.poll(DelayQueue.java:259) at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:323) at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$$Lambda$164/1053864765.run(Unknown Source) at java.lang.Thread.run(Thread.java:750) "AsyncConn Chore Service.Chore.1" #2744 daemon prio=5 os_prio=0 tid=0x00007fc97c48b000 nid=0x1693 waiting on condition [0x00007fc0fd28f000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007fc6f42ca890> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1093) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:750) "ReadOnlyZKClient-cod-1e3xsc06g8mjm-master1.exait3p.z1did1.b0.cloudera.site:2181,cod-1e3xsc06g8mjm-master0.exait3p.z1did1.b0.cloudera.site:2181,cod-1e3xsc06g8mjm-leader0.exait3p.z1did1.b0.cloudera.site:2181@0x64078b62" #2724 daemon prio=5 os_prio=0 tid=0x00007fc97c219000 nid=0x1673 waiting on condition [0x00007fc0fe5a2000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007fc6f42d0d00> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.DelayQueue.poll(DelayQueue.java:259) at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:323) at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$$Lambda$164/1053864765.run(Unknown Source) at java.lang.Thread.run(Thread.java:750) "AsyncConn Chore Service.Chore.1" #2723 daemon prio=5 os_prio=0 tid=0x00007fc97c218000 nid=0x1672 waiting on condition [0x00007fc0fe4a1000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007fc6f42bf5d8> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1093) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:750) "ReadOnlyZKClient-cod-1e3xsc06g8mjm-master1.exait3p.z1did1.b0.cloudera.site:2181,cod-1e3xsc06g8mjm-master0.exait3p.z1did1.b0.cloudera.site:2181,cod-1e3xsc06g8mjm-leader0.exait3p.z1did1.b0.cloudera.site:2181@0x749b95f7" #2690 daemon prio=5 os_prio=0 tid=0x00007fc9848aa000 nid=0x1649 waiting on condition [0x00007fc1001bc000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007fc1e1bf4178> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.DelayQueue.poll(DelayQueue.java:259) at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:323) at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$$Lambda$164/1053864765.run(Unknown Source) at java.lang.Thread.run(Thread.java:750) "ReadOnlyZKClient-cod-1e3xsc06g8mjm-master1.exait3p.z1did1.b0.cloudera.site:2181,cod-1e3xsc06g8mjm-master0.exait3p.z1did1.b0.cloudera.site:2181,cod-1e3xsc06g8mjm-leader0.exait3p.z1did1.b0.cloudera.site:2181@0x0eb856e2" #2689 daemon prio=5 os_prio=0 tid=0x00007fc9848a8800 nid=0x1648 waiting on condition [0x00007fc1003be000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007fc1e1bf2178> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.DelayQueue.poll(DelayQueue.java:259) at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:323) at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$$Lambda$164/1053864765.run(Unknown Source) at java.lang.Thread.run(Thread.java:750) "AsyncConn Chore Service.Chore.1" #2688 daemon prio=5 os_prio=0 tid=0x00007fc984161800 nid=0x1647 waiting on condition [0x00007fc1004bf000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007fc1e1bf6190> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1093) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:750) "AsyncConn Chore Service.Chore.1" #2687 daemon prio=5 os_prio=0 tid=0x00007fc984160800 nid=0x1646 waiting on condition [0x00007fc1005c0000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007fc1e1bf4378> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1093) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:750) "ReadOnlyZKClient-cod-1e3xsc06g8mjm-master1.exait3p.z1did1.b0.cloudera.site:2181,cod-1e3xsc06g8mjm-master0.exait3p.z1did1.b0.cloudera.site:2181,cod-1e3xsc06g8mjm-leader0.exait3p.z1did1.b0.cloudera.site:2181@0x63c4ca3d" #2684 daemon prio=5 os_prio=0 tid=0x00007fc984949800 nid=0x1643 waiting on condition [0x00007fc1008c3000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007fc1e1bf8178> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.DelayQueue.poll(DelayQueue.java:259) at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:323) at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$$Lambda$164/1053864765.run(Unknown Source) at java.lang.Thread.run(Thread.java:750) "AsyncConn Chore Service.Chore.1" #2683 daemon prio=5 os_prio=0 tid=0x00007fc984948800 nid=0x1642 waiting on condition [0x00007fc1009c4000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007fc1e1bf2378> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1093) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:750) "ReadOnlyZKClient-cod-1e3xsc06g8mjm-master1.exait3p.z1did1.b0.cloudera.site:2181,cod-1e3xsc06g8mjm-master0.exait3p.z1did1.b0.cloudera.site:2181,cod-1e3xsc06g8mjm-leader0.exait3p.z1did1.b0.cloudera.site:2181@0x46038c21" #2676 daemon prio=5 os_prio=0 tid=0x00007fc11e194800 nid=0x163b waiting on condition [0x00007fc1016d1000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007fc1e1bfa178> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.DelayQueue.poll(DelayQueue.java:259) at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:323) at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$$Lambda$164/1053864765.run(Unknown Source) at java.lang.Thread.run(Thread.java:750) "AsyncConn Chore Service.Chore.1" #2675 daemon prio=5 os_prio=0 tid=0x00007fc11e192000 nid=0x1638 waiting on condition [0x00007fc101dd8000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007fc1e1bf45b8> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1093) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:750) "ReadOnlyZKClient-cod-1e3xsc06g8mjm-master1.exait3p.z1did1.b0.cloudera.site:2181,cod-1e3xsc06g8mjm-master0.exait3p.z1did1.b0.cloudera.site:2181,cod-1e3xsc06g8mjm-leader0.exait3p.z1did1.b0.cloudera.site:2181@0x0b0ac594" #2672 daemon prio=5 os_prio=0 tid=0x0000558b1419e800 nid=0x1637 waiting on condition [0x00007fc101ed9000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007fc1e1bf63d0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.DelayQueue.poll(DelayQueue.java:259) at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:323) at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$$Lambda$164/1053864765.run(Unknown Source) at java.lang.Thread.run(Thread.java:750) "AsyncConn Chore Service.Chore.1" #2671 daemon prio=5 os_prio=0 tid=0x0000558b1475f800 nid=0x1636 waiting on condition [0x00007fc101fda000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007fc1e1bf25b8> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1093) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:750) "ReadOnlyZKClient-cod-1e3xsc06g8mjm-master1.exait3p.z1did1.b0.cloudera.site:2181,cod-1e3xsc06g8mjm-master0.exait3p.z1did1.b0.cloudera.site:2181,cod-1e3xsc06g8mjm-leader0.exait3p.z1did1.b0.cloudera.site:2181@0x5d0c278a" #2661 daemon prio=5 os_prio=0 tid=0x00007fc984b92800 nid=0x1627 waiting on condition [0x00007fc1027e2000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007fc1e1bf8378> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.DelayQueue.poll(DelayQueue.java:259) at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:323) at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$$Lambda$164/1053864765.run(Unknown Source) at java.lang.Thread.run(Thread.java:750) "AsyncConn Chore Service.Chore.1" #2660 daemon prio=5 os_prio=0 tid=0x00007fc984bd1800 nid=0x1626 waiting on condition [0x00007fc102ce7000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007fc1e1bf47f8> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1093) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:750) "regionserver/cod-1e3xsc06g8mjm-worker124:16020-shortCompactions-1" #2659 daemon prio=5 os_prio=0 tid=0x0000558b146b7000 nid=0x1624 waiting on condition [0x00007fc102ee9000] java.lang.Thread.State: WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007fc1dc090000> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) at java.util.concurrent.PriorityBlockingQueue.take(PriorityBlockingQueue.java:549) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:750) "ReadOnlyZKClient-cod-1e3xsc06g8mjm-master1.exait3p.z1did1.b0.cloudera.site:2181,cod-1e3xsc06g8mjm-master0.exait3p.z1did1.b0.cloudera.site:2181,cod-1e3xsc06g8mjm-leader0.exait3p.z1did1.b0.cloudera.site:2181@0x0f8f12b9" #2647 daemon prio=5 os_prio=0 tid=0x0000558b13dbe000 nid=0x1616 waiting on condition [0x00007fc10530d000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007fc1e1bf27f8> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.DelayQueue.poll(DelayQueue.java:259) at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:323) at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$$Lambda$164/1053864765.run(Unknown Source) at java.lang.Thread.run(Thread.java:750) "AsyncConn Chore Service.Chore.1" #2646 daemon prio=5 os_prio=0 tid=0x0000558b13e40800 nid=0x1615 waiting on condition [0x00007fc10540e000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007fc1e1bf65d0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1093) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:750) "regionserver/cod-1e3xsc06g8mjm-worker124:16020-shortCompactions-0" #2642 daemon prio=5 os_prio=0 tid=0x00007fc97d670800 nid=0x1610 waiting on condition [0x00007fc0fa865000] java.lang.Thread.State: WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007fc1dc090000> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) at java.util.concurrent.PriorityBlockingQueue.take(PriorityBlockingQueue.java:549) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:750) "ReadOnlyZKClient-cod-1e3xsc06g8mjm-master1.exait3p.z1did1.b0.cloudera.site:2181,cod-1e3xsc06g8mjm-master0.exait3p.z1did1.b0.cloudera.site:2181,cod-1e3xsc06g8mjm-leader0.exait3p.z1did1.b0.cloudera.site:2181@0x7363674c" #2632 daemon prio=5 os_prio=0 tid=0x00007fc11de13000 nid=0x15ec waiting on condition [0x00007fc1032ed000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007fc1e1bf8578> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.DelayQueue.poll(DelayQueue.java:259) at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:323) at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$$Lambda$164/1053864765.run(Unknown Source) at java.lang.Thread.run(Thread.java:750) "AsyncConn Chore Service.Chore.1" #2631 daemon prio=5 os_prio=0 tid=0x00007fc11cfbd800 nid=0x15eb waiting on condition [0x00007fc1030eb000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007fc1e1bf29f8> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1093) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:750) "ReadOnlyZKClient-cod-1e3xsc06g8mjm-master1.exait3p.z1did1.b0.cloudera.site:2181,cod-1e3xsc06g8mjm-master0.exait3p.z1did1.b0.cloudera.site:2181,cod-1e3xsc06g8mjm-leader0.exait3p.z1did1.b0.cloudera.site:2181@0x68c41ace" #2613 daemon prio=5 os_prio=0 tid=0x00007fc99bf0e000 nid=0x15d0 waiting on condition [0x00007fc0fab68000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007fc668714e70> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.DelayQueue.poll(DelayQueue.java:259) at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:323) at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$$Lambda$164/1053864765.run(Unknown Source) at java.lang.Thread.run(Thread.java:750) "AsyncConn Chore Service.Chore.1" #2612 daemon prio=5 os_prio=0 tid=0x00007fc11dc6d800 nid=0x15cf waiting on condition [0x00007fc108618000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007fc668727da8> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1093) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:750) "ReadOnlyZKClient-cod-1e3xsc06g8mjm-master1.exait3p.z1did1.b0.cloudera.site:2181,cod-1e3xsc06g8mjm-master0.exait3p.z1did1.b0.cloudera.site:2181,cod-1e3xsc06g8mjm-leader0.exait3p.z1did1.b0.cloudera.site:2181@0x2fc2074f" #2609 daemon prio=5 os_prio=0 tid=0x00007fc98444f800 nid=0x15cc waiting on condition [0x00007fc0fc27f000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007fc668715070> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.DelayQueue.poll(DelayQueue.java:259) at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:323) at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$$Lambda$164/1053864765.run(Unknown Source) at java.lang.Thread.run(Thread.java:750) "AsyncConn Chore Service.Chore.1" #2608 daemon prio=5 os_prio=0 tid=0x00007fc9853b7000 nid=0x15cb waiting on condition [0x00007fc0fc683000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007fc668715270> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1093) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:750) "ReadOnlyZKClient-cod-1e3xsc06g8mjm-master1.exait3p.z1did1.b0.cloudera.site:2181,cod-1e3xsc06g8mjm-master0.exait3p.z1did1.b0.cloudera.site:2181,cod-1e3xsc06g8mjm-leader0.exait3p.z1did1.b0.cloudera.site:2181@0x4e298746" #2601 daemon prio=5 os_prio=0 tid=0x00007fc984c13800 nid=0x15c0 waiting on condition [0x00007fc0fa663000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007fc668727fe8> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.DelayQueue.poll(DelayQueue.java:259) at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:323) at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$$Lambda$164/1053864765.run(Unknown Source) at java.lang.Thread.run(Thread.java:750) "AsyncConn Chore Service.Chore.1" #2600 daemon prio=5 os_prio=0 tid=0x00007fc984de6000 nid=0x15bf waiting on condition [0x00007fc0f9754000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007fc6687154b0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1093) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:750) "ReadOnlyZKClient-cod-1e3xsc06g8mjm-master1.exait3p.z1did1.b0.cloudera.site:2181,cod-1e3xsc06g8mjm-master0.exait3p.z1did1.b0.cloudera.site:2181,cod-1e3xsc06g8mjm-leader0.exait3p.z1did1.b0.cloudera.site:2181@0x484291ca" #2594 daemon prio=5 os_prio=0 tid=0x00007fc97daed800 nid=0x15b8 waiting on condition [0x00007fc0fa562000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007fc6687281e8> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.DelayQueue.poll(DelayQueue.java:259) at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:323) at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$$Lambda$164/1053864765.run(Unknown Source) at java.lang.Thread.run(Thread.java:750) "AsyncConn Chore Service.Chore.1" #2593 daemon prio=5 os_prio=0 tid=0x00007fc97c728800 nid=0x15b7 waiting on condition [0x00007fc10e984000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007fc66872b778> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1093) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:750) "ReadOnlyZKClient-cod-1e3xsc06g8mjm-master1.exait3p.z1did1.b0.cloudera.site:2181,cod-1e3xsc06g8mjm-master0.exait3p.z1did1.b0.cloudera.site:2181,cod-1e3xsc06g8mjm-leader0.exait3p.z1did1.b0.cloudera.site:2181@0x270d5454" #2592 daemon prio=5 os_prio=0 tid=0x00007fc11d139800 nid=0x15b6 waiting on condition [0x00007fc0f9653000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007fc6687283e8> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.DelayQueue.poll(DelayQueue.java:259) at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:323) at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$$Lambda$164/1053864765.run(Unknown Source) at java.lang.Thread.run(Thread.java:750) "AsyncConn Chore Service.Chore.1" #2591 daemon prio=5 os_prio=0 tid=0x00007fc11d289800 nid=0x15b5 waiting on condition [0x00007fc0f9451000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007fc6687285e8> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1093) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:750) "ReadOnlyZKClient-cod-1e3xsc06g8mjm-master1.exait3p.z1did1.b0.cloudera.site:2181,cod-1e3xsc06g8mjm-master0.exait3p.z1did1.b0.cloudera.site:2181,cod-1e3xsc06g8mjm-leader0.exait3p.z1did1.b0.cloudera.site:2181@0x00a1b7e2" #2568 daemon prio=5 os_prio=0 tid=0x0000558b146d5000 nid=0x1512 waiting on condition [0x00007fc1031ec000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007fc66872b9b8> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.DelayQueue.poll(DelayQueue.java:259) at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:323) at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$$Lambda$164/1053864765.run(Unknown Source) at java.lang.Thread.run(Thread.java:750) "AsyncConn Chore Service.Chore.1" #2567 daemon prio=5 os_prio=0 tid=0x0000558b14698800 nid=0x1511 waiting on condition [0x00007fc109929000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007fc66872bbb8> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1093) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:750) "ReadOnlyZKClient-cod-1e3xsc06g8mjm-master1.exait3p.z1did1.b0.cloudera.site:2181,cod-1e3xsc06g8mjm-master0.exait3p.z1did1.b0.cloudera.site:2181,cod-1e3xsc06g8mjm-leader0.exait3p.z1did1.b0.cloudera.site:2181@0x1d6044ee" #2546 daemon prio=5 os_prio=0 tid=0x00007fc11d12f000 nid=0x14c1 waiting on condition [0x00007fc0f9a57000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007fc6a6238978> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.DelayQueue.poll(DelayQueue.java:259) at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:323) at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$$Lambda$164/1053864765.run(Unknown Source) at java.lang.Thread.run(Thread.java:750) "AsyncConn Chore Service.Chore.1" #2545 daemon prio=5 os_prio=0 tid=0x00007fc11d12e800 nid=0x14c0 waiting on condition [0x00007fc0f9b58000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007fc6a6238b78> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1093) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:750) "ReadOnlyZKClient-cod-1e3xsc06g8mjm-master1.exait3p.z1did1.b0.cloudera.site:2181,cod-1e3xsc06g8mjm-master0.exait3p.z1did1.b0.cloudera.site:2181,cod-1e3xsc06g8mjm-leader0.exait3p.z1did1.b0.cloudera.site:2181@0x547cdc37" #2537 daemon prio=5 os_prio=0 tid=0x00007fc97d672000 nid=0x1485 waiting on condition [0x00007fc0fa25f000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007fc6a6238db8> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.DelayQueue.poll(DelayQueue.java:259) at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:323) at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$$Lambda$164/1053864765.run(Unknown Source) at java.lang.Thread.run(Thread.java:750) "AsyncConn Chore Service.Chore.1" #2536 daemon prio=5 os_prio=0 tid=0x00007fc97c19b000 nid=0x1484 waiting on condition [0x00007fc0fa360000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007fc6a6238fb8> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1093) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:750) "ReadOnlyZKClient-cod-1e3xsc06g8mjm-master1.exait3p.z1did1.b0.cloudera.site:2181,cod-1e3xsc06g8mjm-master0.exait3p.z1did1.b0.cloudera.site:2181,cod-1e3xsc06g8mjm-leader0.exait3p.z1did1.b0.cloudera.site:2181@0x06ea2318" #2516 daemon prio=5 os_prio=0 tid=0x00007fc97c92b800 nid=0x1463 waiting on condition [0x00007fc0faf6c000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007fc1defc0378> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.DelayQueue.poll(DelayQueue.java:259) at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:323) at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$$Lambda$164/1053864765.run(Unknown Source) at java.lang.Thread.run(Thread.java:750) "AsyncConn Chore Service.Chore.1" #2515 daemon prio=5 os_prio=0 tid=0x00007fc97c92a000 nid=0x1462 waiting on condition [0x00007fc0fb06d000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007fc1defc2378> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1093) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:750) "RS_OPEN_REGION-regionserver/cod-1e3xsc06g8mjm-worker124:16020-17" #2514 daemon prio=5 os_prio=0 tid=0x00007fc97c929000 nid=0x1461 waiting on condition [0x00007fc0fb16e000] java.lang.Thread.State: WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007fc1dc0901e0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:750) "ReadOnlyZKClient-cod-1e3xsc06g8mjm-master1.exait3p.z1did1.b0.cloudera.site:2181,cod-1e3xsc06g8mjm-master0.exait3p.z1did1.b0.cloudera.site:2181,cod-1e3xsc06g8mjm-leader0.exait3p.z1did1.b0.cloudera.site:2181@0x0f4d33eb" #2503 daemon prio=5 os_prio=0 tid=0x00007fc11db14800 nid=0x1452 waiting on condition [0x00007fc0fbb78000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007fc1defc25b8> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.DelayQueue.poll(DelayQueue.java:259) at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:323) at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$$Lambda$164/1053864765.run(Unknown Source) at java.lang.Thread.run(Thread.java:750) "AsyncConn Chore Service.Chore.1" #2502 daemon prio=5 os_prio=0 tid=0x00007fc11db13800 nid=0x1451 waiting on condition [0x00007fc0fbc79000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007fc1defc0720> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1093) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:750) "ReadOnlyZKClient-cod-1e3xsc06g8mjm-master1.exait3p.z1did1.b0.cloudera.site:2181,cod-1e3xsc06g8mjm-master0.exait3p.z1did1.b0.cloudera.site:2181,cod-1e3xsc06g8mjm-leader0.exait3p.z1did1.b0.cloudera.site:2181@0x69d9b524" #2499 daemon prio=5 os_prio=0 tid=0x0000558b13d1c000 nid=0x144e waiting on condition [0x00007fc0fbf7c000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007fc1defc27b8> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.DelayQueue.poll(DelayQueue.java:259) at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:323) at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$$Lambda$164/1053864765.run(Unknown Source) at java.lang.Thread.run(Thread.java:750) "AsyncConn Chore Service.Chore.1" #2498 daemon prio=5 os_prio=0 tid=0x0000558b13d1a800 nid=0x144d waiting on condition [0x00007fc0fc07d000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007fc1defc0960> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1093) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:750) "RS_OPEN_REGION-regionserver/cod-1e3xsc06g8mjm-worker124:16020-16" #2497 daemon prio=5 os_prio=0 tid=0x0000558b1417a800 nid=0x144c waiting on condition [0x00007fc0fc17e000] java.lang.Thread.State: WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007fc1dc0901e0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:750) "RS_OPEN_REGION-regionserver/cod-1e3xsc06g8mjm-worker124:16020-15" #2496 daemon prio=5 os_prio=0 tid=0x0000558b1475e000 nid=0x144b waiting on condition [0x00007fc0fc380000] java.lang.Thread.State: WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007fc1dc0901e0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:750) "ReadOnlyZKClient-cod-1e3xsc06g8mjm-master1.exait3p.z1did1.b0.cloudera.site:2181,cod-1e3xsc06g8mjm-master0.exait3p.z1did1.b0.cloudera.site:2181,cod-1e3xsc06g8mjm-leader0.exait3p.z1did1.b0.cloudera.site:2181@0x13d0330f" #2481 daemon prio=5 os_prio=0 tid=0x00007fc9847ed000 nid=0x141e waiting on condition [0x00007fc0fcb88000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007fc1defc2b60> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.DelayQueue.poll(DelayQueue.java:259) at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:323) at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$$Lambda$164/1053864765.run(Unknown Source) at java.lang.Thread.run(Thread.java:750) "AsyncConn Chore Service.Chore.1" #2480 daemon prio=5 os_prio=0 tid=0x00007fc9847f3800 nid=0x141d waiting on condition [0x00007fc0fcc89000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007fc1defc0d48> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1093) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:750) "RS_OPEN_REGION-regionserver/cod-1e3xsc06g8mjm-worker124:16020-14" #2479 daemon prio=5 os_prio=0 tid=0x00007fc11d158800 nid=0x141c waiting on condition [0x00007fc0fcd8a000] java.lang.Thread.State: WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007fc1dc0901e0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:750) "ReadOnlyZKClient-cod-1e3xsc06g8mjm-master1.exait3p.z1did1.b0.cloudera.site:2181,cod-1e3xsc06g8mjm-master0.exait3p.z1did1.b0.cloudera.site:2181,cod-1e3xsc06g8mjm-leader0.exait3p.z1did1.b0.cloudera.site:2181@0x52941867" #2450 daemon prio=5 os_prio=0 tid=0x00007fc11de06800 nid=0x13f4 waiting on condition [0x00007fc0fe29f000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007fc1de543578> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.DelayQueue.poll(DelayQueue.java:259) at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:323) at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$$Lambda$164/1053864765.run(Unknown Source) at java.lang.Thread.run(Thread.java:750) "AsyncConn Chore Service.Chore.1" #2449 daemon prio=5 os_prio=0 tid=0x00007fc11de02000 nid=0x13f3 waiting on condition [0x00007fc0fe3a0000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007fc1de5499b0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1093) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:750) "ReadOnlyZKClient-cod-1e3xsc06g8mjm-master1.exait3p.z1did1.b0.cloudera.site:2181,cod-1e3xsc06g8mjm-master0.exait3p.z1did1.b0.cloudera.site:2181,cod-1e3xsc06g8mjm-leader0.exait3p.z1did1.b0.cloudera.site:2181@0x552bac7c" #2446 daemon prio=5 os_prio=0 tid=0x00007fc984932800 nid=0x13f0 waiting on condition [0x00007fc0fe6a3000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007fc1de541578> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.DelayQueue.poll(DelayQueue.java:259) at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:323) at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$$Lambda$164/1053864765.run(Unknown Source) at java.lang.Thread.run(Thread.java:750) "AsyncConn Chore Service.Chore.1" #2445 daemon prio=5 os_prio=0 tid=0x00007fc984931000 nid=0x13ef waiting on condition [0x00007fc0fe7a4000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007fc1de543778> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1093) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:750) "RS_OPEN_REGION-regionserver/cod-1e3xsc06g8mjm-worker124:16020-13" #2444 daemon prio=5 os_prio=0 tid=0x00007fc984857800 nid=0x13ee waiting on condition [0x00007fc0feba8000] java.lang.Thread.State: WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007fc1dc0901e0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:750) "RS_OPEN_REGION-regionserver/cod-1e3xsc06g8mjm-worker124:16020-12" #2443 daemon prio=5 os_prio=0 tid=0x00007fc984856800 nid=0x13ed waiting on condition [0x00007fc102de8000] java.lang.Thread.State: WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007fc1dc0901e0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:750) "ReadOnlyZKClient-cod-1e3xsc06g8mjm-master1.exait3p.z1did1.b0.cloudera.site:2181,cod-1e3xsc06g8mjm-master0.exait3p.z1did1.b0.cloudera.site:2181,cod-1e3xsc06g8mjm-leader0.exait3p.z1did1.b0.cloudera.site:2181@0x1204ced0" #2431 daemon prio=5 os_prio=0 tid=0x0000558b146c5800 nid=0x13dc waiting on condition [0x00007fc0ff1ae000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007fc1de549bf0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.DelayQueue.poll(DelayQueue.java:259) at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:323) at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$$Lambda$164/1053864765.run(Unknown Source) at java.lang.Thread.run(Thread.java:750) "AsyncConn Chore Service.Chore.1" #2430 daemon prio=5 os_prio=0 tid=0x0000558b146c4000 nid=0x13db waiting on condition [0x00007fc0ff2af000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007fc1de541778> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1093) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:750) "RS_OPEN_REGION-regionserver/cod-1e3xsc06g8mjm-worker124:16020-11" #2429 daemon prio=5 os_prio=0 tid=0x00007fc97cb8e800 nid=0x13da waiting on condition [0x00007fc0ff3b0000] java.lang.Thread.State: WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007fc1dc0901e0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:750) "ReadOnlyZKClient-cod-1e3xsc06g8mjm-master1.exait3p.z1did1.b0.cloudera.site:2181,cod-1e3xsc06g8mjm-master0.exait3p.z1did1.b0.cloudera.site:2181,cod-1e3xsc06g8mjm-leader0.exait3p.z1did1.b0.cloudera.site:2181@0x40bd6cd7" #2421 daemon prio=5 os_prio=0 tid=0x00007fc9845a5000 nid=0x13d0 waiting on condition [0x00007fc0ffbb8000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007fc1de549df0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.DelayQueue.poll(DelayQueue.java:259) at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:323) at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$$Lambda$164/1053864765.run(Unknown Source) at java.lang.Thread.run(Thread.java:750) "AsyncConn Chore Service.Chore.1" #2420 daemon prio=5 os_prio=0 tid=0x00007fc9849f0800 nid=0x13cf waiting on condition [0x00007fc0ffcb9000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007fc1de5419b8> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1093) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:750) "RS_OPEN_REGION-regionserver/cod-1e3xsc06g8mjm-worker124:16020-10" #2419 daemon prio=5 os_prio=0 tid=0x0000558b14173000 nid=0x13ce waiting on condition [0x00007fc0ffdba000] java.lang.Thread.State: WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007fc1dc0901e0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:750) "ReadOnlyZKClient-cod-1e3xsc06g8mjm-master1.exait3p.z1did1.b0.cloudera.site:2181,cod-1e3xsc06g8mjm-master0.exait3p.z1did1.b0.cloudera.site:2181,cod-1e3xsc06g8mjm-leader0.exait3p.z1did1.b0.cloudera.site:2181@0x33c81c3d" #2400 daemon prio=5 os_prio=0 tid=0x00007fc97c7fa800 nid=0x13ae waiting on condition [0x00007fc100cc7000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007fc1ddb5d378> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.DelayQueue.poll(DelayQueue.java:259) at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:323) at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$$Lambda$164/1053864765.run(Unknown Source) at java.lang.Thread.run(Thread.java:750) "AsyncConn Chore Service.Chore.1" #2399 daemon prio=5 os_prio=0 tid=0x00007fc97c7f9000 nid=0x13ad waiting on condition [0x00007fc100dc8000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007fc1ddb59378> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1093) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:750) "RS_OPEN_REGION-regionserver/cod-1e3xsc06g8mjm-worker124:16020-9" #2398 daemon prio=5 os_prio=0 tid=0x0000558b14a27000 nid=0x13ac waiting on condition [0x00007fc100ec9000] java.lang.Thread.State: WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007fc1dc0901e0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:750) "ReadOnlyZKClient-cod-1e3xsc06g8mjm-master1.exait3p.z1did1.b0.cloudera.site:2181,cod-1e3xsc06g8mjm-master0.exait3p.z1did1.b0.cloudera.site:2181,cod-1e3xsc06g8mjm-leader0.exait3p.z1did1.b0.cloudera.site:2181@0x7d335c87" #2393 daemon prio=5 os_prio=0 tid=0x00007fc984489800 nid=0x13a5 waiting on condition [0x00007fc1013ce000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007fc1ddb61378> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.DelayQueue.poll(DelayQueue.java:259) at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:323) at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$$Lambda$164/1053864765.run(Unknown Source) at java.lang.Thread.run(Thread.java:750) "AsyncConn Chore Service.Chore.1" #2392 daemon prio=5 os_prio=0 tid=0x00007fc984488000 nid=0x13a4 waiting on condition [0x00007fc1014cf000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007fc1ddb65378> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1093) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:750) "RS_OPEN_REGION-regionserver/cod-1e3xsc06g8mjm-worker124:16020-8" #2391 daemon prio=5 os_prio=0 tid=0x00007fc11de38800 nid=0x13a3 waiting on condition [0x00007fc1015d0000] java.lang.Thread.State: WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007fc1dc0901e0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:750) "ReadOnlyZKClient-cod-1e3xsc06g8mjm-master1.exait3p.z1did1.b0.cloudera.site:2181,cod-1e3xsc06g8mjm-master0.exait3p.z1did1.b0.cloudera.site:2181,cod-1e3xsc06g8mjm-leader0.exait3p.z1did1.b0.cloudera.site:2181@0x3ab34b67" #2386 daemon prio=5 os_prio=0 tid=0x00007fc11dd91000 nid=0x139d waiting on condition [0x00007fc101ad5000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007fc1ddb595b8> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.DelayQueue.poll(DelayQueue.java:259) at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:323) at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$$Lambda$164/1053864765.run(Unknown Source) at java.lang.Thread.run(Thread.java:750) "AsyncConn Chore Service.Chore.1" #2385 daemon prio=5 os_prio=0 tid=0x00007fc11dde5800 nid=0x139c waiting on condition [0x00007fc101bd6000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007fc1ddb655b8> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1093) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:750) "RS_OPEN_REGION-regionserver/cod-1e3xsc06g8mjm-worker124:16020-7" #2384 daemon prio=5 os_prio=0 tid=0x00007fc984e51800 nid=0x139b waiting on condition [0x00007fc101cd7000] java.lang.Thread.State: WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007fc1dc0901e0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:750) "ReadOnlyZKClient-cod-1e3xsc06g8mjm-master1.exait3p.z1did1.b0.cloudera.site:2181,cod-1e3xsc06g8mjm-master0.exait3p.z1did1.b0.cloudera.site:2181,cod-1e3xsc06g8mjm-leader0.exait3p.z1did1.b0.cloudera.site:2181@0x5be93842" #2374 daemon prio=5 os_prio=0 tid=0x00007fc11cf8d800 nid=0x138d waiting on condition [0x00007fc1024df000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007fc1ddb67520> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.DelayQueue.poll(DelayQueue.java:259) at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:323) at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$$Lambda$164/1053864765.run(Unknown Source) at java.lang.Thread.run(Thread.java:750) "AsyncConn Chore Service.Chore.1" #2373 daemon prio=5 os_prio=0 tid=0x00007fc11d0db800 nid=0x138c waiting on condition [0x00007fc1025e0000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007fc1ddb5f378> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1093) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:750) "ReadOnlyZKClient-cod-1e3xsc06g8mjm-master1.exait3p.z1did1.b0.cloudera.site:2181,cod-1e3xsc06g8mjm-master0.exait3p.z1did1.b0.cloudera.site:2181,cod-1e3xsc06g8mjm-leader0.exait3p.z1did1.b0.cloudera.site:2181@0x380c223b" #2370 daemon prio=5 os_prio=0 tid=0x00007fc984052000 nid=0x1389 waiting on condition [0x00007fc1028e3000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007fc1ddb63520> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.DelayQueue.poll(DelayQueue.java:259) at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:323) at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$$Lambda$164/1053864765.run(Unknown Source) at java.lang.Thread.run(Thread.java:750) "AsyncConn Chore Service.Chore.1" #2369 daemon prio=5 os_prio=0 tid=0x00007fc984050800 nid=0x1388 waiting on condition [0x00007fc1029e4000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007fc1ddb657f8> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1093) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:750) "RS_OPEN_REGION-regionserver/cod-1e3xsc06g8mjm-worker124:16020-6" #2368 daemon prio=5 os_prio=0 tid=0x00007fc97c45a000 nid=0x1387 waiting on condition [0x00007fc102ae5000] java.lang.Thread.State: WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007fc1dc0901e0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:750) "RS_OPEN_REGION-regionserver/cod-1e3xsc06g8mjm-worker124:16020-5" #2367 daemon prio=5 os_prio=0 tid=0x00007fc97ce46000 nid=0x1386 waiting on condition [0x00007fc102be6000] java.lang.Thread.State: WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007fc1dc0901e0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:750) "ReadOnlyZKClient-cod-1e3xsc06g8mjm-master1.exait3p.z1did1.b0.cloudera.site:2181,cod-1e3xsc06g8mjm-master0.exait3p.z1did1.b0.cloudera.site:2181,cod-1e3xsc06g8mjm-leader0.exait3p.z1did1.b0.cloudera.site:2181@0x3f5ca06a" #2351 daemon prio=5 os_prio=0 tid=0x00007fc99b396000 nid=0x1349 waiting on condition [0x00007fc1033ee000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007fc1dcf90618> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.DelayQueue.poll(DelayQueue.java:259) at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:323) at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$$Lambda$164/1053864765.run(Unknown Source) at java.lang.Thread.run(Thread.java:750) "AsyncConn Chore Service.Chore.1" #2350 daemon prio=5 os_prio=0 tid=0x00007fc99b394800 nid=0x1348 waiting on condition [0x00007fc1034ef000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007fc1dcf95a90> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1093) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:750) "RS_OPEN_REGION-regionserver/cod-1e3xsc06g8mjm-worker124:16020-4" #2348 daemon prio=5 os_prio=0 tid=0x00007fc9841d7000 nid=0x1346 waiting on condition [0x00007fc1036f1000] java.lang.Thread.State: WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007fc1dc0901e0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:750) "hfile-prefetch-1668707571454" #2346 daemon prio=5 os_prio=0 tid=0x0000558b13f0c000 nid=0x1344 waiting on condition [0x00007fc1039f2000] java.lang.Thread.State: WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007fc1dd3753c0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1081) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:750) "ReadOnlyZKClient-cod-1e3xsc06g8mjm-master1.exait3p.z1did1.b0.cloudera.site:2181,cod-1e3xsc06g8mjm-master0.exait3p.z1did1.b0.cloudera.site:2181,cod-1e3xsc06g8mjm-leader0.exait3p.z1did1.b0.cloudera.site:2181@0x696d94c7" #2335 daemon prio=5 os_prio=0 tid=0x0000558b13bfe000 nid=0x1332 waiting on condition [0x00007fc104600000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007fc1dcf95cd0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.DelayQueue.poll(DelayQueue.java:259) at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:323) at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$$Lambda$164/1053864765.run(Unknown Source) at java.lang.Thread.run(Thread.java:750) "AsyncConn Chore Service.Chore.1" #2334 daemon prio=5 os_prio=0 tid=0x0000558b13bfc800 nid=0x1331 waiting on condition [0x00007fc104701000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007fc1dcf909c0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1093) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:750) "RS_OPEN_REGION-regionserver/cod-1e3xsc06g8mjm-worker124:16020-3" #2333 daemon prio=5 os_prio=0 tid=0x00007fc97c936800 nid=0x1330 waiting on condition [0x00007fc104802000] java.lang.Thread.State: WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007fc1dc0901e0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:750) "ReadOnlyZKClient-cod-1e3xsc06g8mjm-master1.exait3p.z1did1.b0.cloudera.site:2181,cod-1e3xsc06g8mjm-master0.exait3p.z1did1.b0.cloudera.site:2181,cod-1e3xsc06g8mjm-leader0.exait3p.z1did1.b0.cloudera.site:2181@0x56298df9" #2328 daemon prio=5 os_prio=0 tid=0x00007fc97c68d000 nid=0x132a waiting on condition [0x00007fc104d07000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007fc1dcf90c00> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.DelayQueue.poll(DelayQueue.java:259) at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:323) at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$$Lambda$164/1053864765.run(Unknown Source) at java.lang.Thread.run(Thread.java:750) "AsyncConn Chore Service.Chore.1" #2327 daemon prio=5 os_prio=0 tid=0x00007fc97c68b000 nid=0x1329 waiting on condition [0x00007fc104e08000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007fc1dcf96078> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1093) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:750) "RS_OPEN_REGION-regionserver/cod-1e3xsc06g8mjm-worker124:16020-2" #2326 daemon prio=5 os_prio=0 tid=0x0000558b13ca6800 nid=0x1328 waiting on condition [0x00007fc104f09000] java.lang.Thread.State: WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007fc1dc0901e0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:750) "ReadOnlyZKClient-cod-1e3xsc06g8mjm-master1.exait3p.z1did1.b0.cloudera.site:2181,cod-1e3xsc06g8mjm-master0.exait3p.z1did1.b0.cloudera.site:2181,cod-1e3xsc06g8mjm-leader0.exait3p.z1did1.b0.cloudera.site:2181@0x4777a621" #2320 daemon prio=5 os_prio=0 tid=0x00007fc984a1d800 nid=0x1320 waiting on condition [0x00007fc10550f000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007fc1dcf962b8> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.DelayQueue.poll(DelayQueue.java:259) at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:323) at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$$Lambda$164/1053864765.run(Unknown Source) at java.lang.Thread.run(Thread.java:750) "ReadOnlyZKClient-cod-1e3xsc06g8mjm-master1.exait3p.z1did1.b0.cloudera.site:2181,cod-1e3xsc06g8mjm-master0.exait3p.z1did1.b0.cloudera.site:2181,cod-1e3xsc06g8mjm-leader0.exait3p.z1did1.b0.cloudera.site:2181@0x23b546a8" #2319 daemon prio=5 os_prio=0 tid=0x00007fc11c08b000 nid=0x131f waiting on condition [0x00007fc105610000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007fc1dcf90fa8> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.DelayQueue.poll(DelayQueue.java:259) at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:323) at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$$Lambda$164/1053864765.run(Unknown Source) at java.lang.Thread.run(Thread.java:750) "AsyncConn Chore Service.Chore.1" #2318 daemon prio=5 os_prio=0 tid=0x00007fc984a1c000 nid=0x131e waiting on condition [0x00007fc105711000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007fc1dcf964b8> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1093) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:750) "AsyncConn Chore Service.Chore.1" #2317 daemon prio=5 os_prio=0 tid=0x00007fc99be9a800 nid=0x131d waiting on condition [0x00007fc108014000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007fc1dcf911a8> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1093) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:750) "RS_OPEN_REGION-regionserver/cod-1e3xsc06g8mjm-worker124:16020-1" #2316 daemon prio=5 os_prio=0 tid=0x00007fc99b601800 nid=0x131c waiting on condition [0x00007fc10e883000] java.lang.Thread.State: WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007fc1dc0901e0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:750) "RS_OPEN_REGION-regionserver/cod-1e3xsc06g8mjm-worker124:16020-0" #2315 daemon prio=5 os_prio=0 tid=0x00007fc11cddf800 nid=0x131b waiting on condition [0x00007fc10ea85000] java.lang.Thread.State: WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007fc1dc0901e0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:750) "RS-EventLoopGroup-1-4" #2306 daemon prio=10 os_prio=0 tid=0x00007fc98430d000 nid=0x1312 runnable [0x00007fc108517000] java.lang.Thread.State: RUNNABLE at org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) at org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:157) at org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:150) at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:290) at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:347) at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:989) at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) at java.lang.Thread.run(Thread.java:750) "RS_REFRESH_PEER-regionserver/cod-1e3xsc06g8mjm-worker124:16020-0" #2269 daemon prio=5 os_prio=0 tid=0x00007fc9847b0800 nid=0xf48 waiting on condition [0x00007fc109525000] java.lang.Thread.State: WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007fc1dc07c018> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:750) "Monitor thread for TaskMonitor" #2268 daemon prio=5 os_prio=0 tid=0x00007fc9842fe000 nid=0xf47 waiting on condition [0x00007fc109727000] java.lang.Thread.State: TIMED_WAITING (sleeping) at java.lang.Thread.sleep(Native Method) at org.apache.hadoop.hbase.monitoring.TaskMonitor$MonitorRunnable.run(TaskMonitor.java:302) at java.lang.Thread.run(Thread.java:750) "RS-EventLoopGroup-1-3" #2267 daemon prio=10 os_prio=0 tid=0x00007fc9842eb000 nid=0xf46 runnable [0x00007fc109626000] java.lang.Thread.State: RUNNABLE at org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) at org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:157) at org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:150) at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:290) at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:347) at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:989) at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) at java.lang.Thread.run(Thread.java:750) "Connector-Scheduler-690eda7a-1" #717 prio=5 os_prio=0 tid=0x00007fc98431c800 nid=0x7590 waiting on condition [0x00007fc10b23a000] java.lang.Thread.State: WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007fc1c9fe5d70> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1081) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:750) "Keep-Alive-Timer" #692 daemon prio=8 os_prio=0 tid=0x0000558b140d1800 nid=0x750c waiting on condition [0x00007fc108115000] java.lang.Thread.State: TIMED_WAITING (sleeping) at java.lang.Thread.sleep(Native Method) at sun.net.www.http.KeepAliveCache.run(KeepAliveCache.java:172) at java.lang.Thread.run(Thread.java:750) "org.apache.hadoop.hdfs.PeerCache@7dd52d46" #669 daemon prio=5 os_prio=0 tid=0x0000558b140e2000 nid=0x74b8 waiting on condition [0x00007fc108216000] java.lang.Thread.State: TIMED_WAITING (sleeping) at java.lang.Thread.sleep(Native Method) at org.apache.hadoop.hdfs.PeerCache.run(PeerCache.java:253) at org.apache.hadoop.hdfs.PeerCache.access$000(PeerCache.java:46) at org.apache.hadoop.hdfs.PeerCache$1.run(PeerCache.java:124) at java.lang.Thread.run(Thread.java:750) "LeaseRenewer:hbase@ns1" #664 daemon prio=5 os_prio=0 tid=0x00007fc97d628000 nid=0x74af waiting on condition [0x00007fc108a1a000] java.lang.Thread.State: TIMED_WAITING (sleeping) at java.lang.Thread.sleep(Native Method) at org.apache.hadoop.hdfs.client.impl.LeaseRenewer.run(LeaseRenewer.java:412) at org.apache.hadoop.hdfs.client.impl.LeaseRenewer.access$600(LeaseRenewer.java:76) at org.apache.hadoop.hdfs.client.impl.LeaseRenewer$1.run(LeaseRenewer.java:308) at java.lang.Thread.run(Thread.java:750) "sync.4" #661 prio=5 os_prio=0 tid=0x00007fc11c6e4800 nid=0x74aa waiting on condition [0x00007fc108c1c000] java.lang.Thread.State: WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007fc1dc0983e8> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:629) "sync.3" #660 prio=5 os_prio=0 tid=0x00007fc11c6e2800 nid=0x74a9 waiting on condition [0x00007fc108d1d000] java.lang.Thread.State: WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007fc1dc07e730> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:629) "sync.2" #659 prio=5 os_prio=0 tid=0x00007fc11c6e1000 nid=0x74a8 waiting on condition [0x00007fc108e1e000] java.lang.Thread.State: WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007fc1dc07c408> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:629) "sync.1" #658 prio=5 os_prio=0 tid=0x00007fc11c6df800 nid=0x74a7 waiting on condition [0x00007fc108f1f000] java.lang.Thread.State: WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007fc1dc0985e0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:629) "sync.0" #657 prio=5 os_prio=0 tid=0x00007fc11c6df000 nid=0x74a6 waiting on condition [0x00007fc109020000] java.lang.Thread.State: WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007fc1dc078730> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:629) "regionserver/cod-1e3xsc06g8mjm-worker124:16020.append-pool-0" #662 daemon prio=5 os_prio=0 tid=0x00007fc97c525800 nid=0x74a5 waiting on condition [0x00007fc109c2c000] java.lang.Thread.State: WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007fc1dc07e910> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) at com.lmax.disruptor.BlockingWaitStrategy.waitFor(BlockingWaitStrategy.java:45) at com.lmax.disruptor.ProcessingSequenceBarrier.waitFor(ProcessingSequenceBarrier.java:56) at com.lmax.disruptor.BatchEventProcessor.run(BatchEventProcessor.java:124) at java.lang.Thread.run(Thread.java:750) "cod-1e3xsc06g8mjm-worker124:16020Replication Statistics #0" #272 daemon prio=5 os_prio=0 tid=0x00007fc97db8a800 nid=0x7492 waiting on condition [0x00007fc109a2a000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007fc1dc078910> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1093) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:750) "ReplicationExecutor-0" #271 daemon prio=5 os_prio=0 tid=0x00007fc97c666000 nid=0x7491 waiting on condition [0x00007fc109b2b000] java.lang.Thread.State: WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007fc1dce73900> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:750) "regionserver/cod-1e3xsc06g8mjm-worker124:16020-MemStoreChunkPool Statistics" #269 daemon prio=5 os_prio=0 tid=0x00007fc97c610000 nid=0x748f waiting on condition [0x00007fc109d2d000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007fc1dc07c5e8> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1093) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:750) "regionserver/cod-1e3xsc06g8mjm-worker124:16020-MemStoreChunkPool Statistics" #267 daemon prio=5 os_prio=0 tid=0x00007fc97db6f000 nid=0x748e waiting on condition [0x00007fc109e2e000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007fc1dc078b50> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1093) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:750) "regionserver/cod-1e3xsc06g8mjm-worker124:16020.leaseChecker" #258 daemon prio=5 os_prio=0 tid=0x00007fc97cd83800 nid=0x748d waiting on condition [0x00007fc109f2f000] java.lang.Thread.State: TIMED_WAITING (sleeping) at java.lang.Thread.sleep(Native Method) at org.apache.hadoop.hbase.regionserver.LeaseManager.run(LeaseManager.java:90) "regionserver/cod-1e3xsc06g8mjm-worker124:16020.procedureResultReporter" #260 daemon prio=5 os_prio=0 tid=0x00007fc97cd82000 nid=0x748c waiting on condition [0x00007fc10a030000] java.lang.Thread.State: WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007fc1dc07c830> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) at org.apache.hadoop.hbase.regionserver.RemoteProcedureResultReporter.run(RemoteProcedureResultReporter.java:77) "MemStoreFlusher.1" #264 prio=5 os_prio=0 tid=0x00007fc97cd80000 nid=0x748b waiting on condition [0x00007fc10a131000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007fc1dc07ed30> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.DelayQueue.poll(DelayQueue.java:268) at java.util.concurrent.DelayQueue.poll(DelayQueue.java:70) at org.apache.hadoop.hbase.regionserver.MemStoreFlusher$FlushHandler.run(MemStoreFlusher.java:335) "MemStoreFlusher.0" #262 prio=5 os_prio=0 tid=0x00007fc97cb65800 nid=0x748a waiting on condition [0x00007fc10a232000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007fc1dc07ed30> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.DelayQueue.poll(DelayQueue.java:268) at java.util.concurrent.DelayQueue.poll(DelayQueue.java:70) at org.apache.hadoop.hbase.regionserver.MemStoreFlusher$FlushHandler.run(MemStoreFlusher.java:335) "regionserver/cod-1e3xsc06g8mjm-worker124:16020.logRoller" #259 daemon prio=5 os_prio=0 tid=0x00007fc97cb64800 nid=0x7489 in Object.wait() [0x00007fc10a333000] java.lang.Thread.State: TIMED_WAITING (on object monitor) at java.lang.Object.wait(Native Method) at org.apache.hadoop.hbase.wal.AbstractWALRoller.run(AbstractWALRoller.java:181) - locked <0x00007fc1dc07c8c0> (a org.apache.hadoop.hbase.regionserver.LogRoller) "regionserver/cod-1e3xsc06g8mjm-worker124:16020.Chore.1" #261 daemon prio=5 os_prio=0 tid=0x00007fc97cb61800 nid=0x7488 waiting on condition [0x00007fc10a434000] java.lang.Thread.State: WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007fc1c9fe5e08> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1088) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:750) "regionserver/cod-1e3xsc06g8mjm-worker124:16020-longCompactions-1" #257 daemon prio=5 os_prio=0 tid=0x00007fc97d88f000 nid=0x7485 waiting on condition [0x00007fc10ab35000] java.lang.Thread.State: WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007fc1dc07eef0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) at org.apache.hadoop.hbase.util.StealJobQueue.take(StealJobQueue.java:106) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:750) "regionserver/cod-1e3xsc06g8mjm-worker124:16020-longCompactions-0" #256 daemon prio=5 os_prio=0 tid=0x00007fc97d88b000 nid=0x7484 waiting on condition [0x00007fc10ac36000] java.lang.Thread.State: WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007fc1dc07eef0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) at org.apache.hadoop.hbase.util.StealJobQueue.take(StealJobQueue.java:106) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:750) "JvmPauseMonitor" #255 daemon prio=5 os_prio=0 tid=0x00007fc97d703800 nid=0x7483 waiting on condition [0x00007fc10ad37000] java.lang.Thread.State: TIMED_WAITING (sleeping) at java.lang.Thread.sleep(Native Method) at org.apache.hadoop.hbase.util.JvmPauseMonitor$Monitor.run(JvmPauseMonitor.java:154) at java.lang.Thread.run(Thread.java:750) "RpcClient-timer-pool-0" #225 daemon prio=5 os_prio=0 tid=0x00007fc984acd000 nid=0x747a sleeping[0x00007fc10c33c000] java.lang.Thread.State: TIMED_WAITING (sleeping) at java.lang.Thread.sleep(Native Method) at org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$Worker.waitForNextTick(HashedWheelTimer.java:569) at org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$Worker.run(HashedWheelTimer.java:465) at java.lang.Thread.run(Thread.java:750) "RS-EventLoopGroup-1-2" #251 daemon prio=10 os_prio=0 tid=0x00007fc97c95c800 nid=0x7478 runnable [0x00007fc10c43d000] java.lang.Thread.State: RUNNABLE at org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) at org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:157) at org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:150) at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:290) at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:347) at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:989) at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) at java.lang.Thread.run(Thread.java:750) "policyDownloadTimer" #250 daemon prio=5 os_prio=0 tid=0x00007fc97d6f6800 nid=0x7477 in Object.wait() [0x00007fc10c73e000] java.lang.Thread.State: TIMED_WAITING (on object monitor) at java.lang.Object.wait(Native Method) at java.util.TimerThread.mainLoop(Timer.java:552) - locked <0x00007fc1dc08c7a8> (a java.util.TaskQueue) at java.util.TimerThread.run(Timer.java:505) "PolicyRefresher(serviceName=cod_exait3-cod-prod_hbase)-241" #241 daemon prio=5 os_prio=0 tid=0x00007fc97d6f8000 nid=0x7476 waiting on condition [0x00007fc10c83f000] java.lang.Thread.State: WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007fc1dc08c9c0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) at org.apache.ranger.plugin.util.PolicyRefresher.run(PolicyRefresher.java:207) "Ranger async Audit cleanup" #240 daemon prio=5 os_prio=0 tid=0x00007fc97cee7800 nid=0x7467 waiting on condition [0x00007fc10d776000] java.lang.Thread.State: WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007fc1dc0795d8> (a java.util.concurrent.Semaphore$NonfairSync) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer.parkAndCheckInterrupt(AbstractQueuedSynchronizer.java:836) at java.util.concurrent.locks.AbstractQueuedSynchronizer.doAcquireSharedInterruptibly(AbstractQueuedSynchronizer.java:997) at java.util.concurrent.locks.AbstractQueuedSynchronizer.acquireSharedInterruptibly(AbstractQueuedSynchronizer.java:1304) at java.util.concurrent.Semaphore.acquire(Semaphore.java:312) at org.apache.ranger.audit.provider.AuditProviderFactory$RangerAsyncAuditCleanup.run(AuditProviderFactory.java:524) at java.lang.Thread.run(Thread.java:750) "org.apache.ranger.audit.queue.AuditAsyncQueue0" #238 daemon prio=5 os_prio=0 tid=0x00007fc97c3a6000 nid=0x7466 waiting on condition [0x00007fc10d877000] java.lang.Thread.State: WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007fc1dc08cba0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) at org.apache.ranger.audit.queue.AuditAsyncQueue.runLogAudit(AuditAsyncQueue.java:143) at org.apache.ranger.audit.queue.AuditAsyncQueue.run(AuditAsyncQueue.java:131) at java.lang.Thread.run(Thread.java:750) "org.apache.ranger.audit.queue.AuditSummaryQueue0" #237 daemon prio=5 os_prio=0 tid=0x00007fc97c3a4000 nid=0x7465 waiting on condition [0x00007fc10d978000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007fc1dc07f4e8> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) at org.apache.ranger.audit.queue.AuditSummaryQueue.runLogAudit(AuditSummaryQueue.java:173) at org.apache.ranger.audit.queue.AuditSummaryQueue.run(AuditSummaryQueue.java:152) at java.lang.Thread.run(Thread.java:750) "org.apache.ranger.audit.queue.AuditBatchQueue1" #236 daemon prio=5 os_prio=0 tid=0x00007fc97dbc1000 nid=0x7464 waiting on condition [0x00007fc10da79000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007fc1dc07f7a8> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.ArrayBlockingQueue.poll(ArrayBlockingQueue.java:418) at org.apache.ranger.audit.queue.AuditBatchQueue.runLogAudit(AuditBatchQueue.java:259) at org.apache.ranger.audit.queue.AuditBatchQueue.run(AuditBatchQueue.java:215) at java.lang.Thread.run(Thread.java:750) "hbaseRegional.async.summary.multi_dest.batch_hbaseRegional.async.summary.multi_dest.batch.solr_destWriter" #235 daemon prio=5 os_prio=0 tid=0x00007fc97dbbf000 nid=0x7463 waiting on condition [0x00007fc10db7a000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007fc1dc08ce78> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) at org.apache.ranger.audit.queue.AuditFileSpool.runLogAudit(AuditFileSpool.java:779) at org.apache.ranger.audit.queue.AuditFileSpool.run(AuditFileSpool.java:757) at java.lang.Thread.run(Thread.java:750) "org.apache.ranger.audit.queue.AuditBatchQueue0" #234 daemon prio=5 os_prio=0 tid=0x00007fc97c3a1800 nid=0x7462 waiting on condition [0x00007fc10dc7b000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007fc1dc0797d8> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.ArrayBlockingQueue.poll(ArrayBlockingQueue.java:418) at org.apache.ranger.audit.queue.AuditBatchQueue.runLogAudit(AuditBatchQueue.java:259) at org.apache.ranger.audit.queue.AuditBatchQueue.run(AuditBatchQueue.java:215) at java.lang.Thread.run(Thread.java:750) "hbaseRegional.async.summary.multi_dest.batch_hbaseRegional.async.summary.multi_dest.batch.hdfs_destWriter" #233 daemon prio=5 os_prio=0 tid=0x00007fc97c1ee800 nid=0x7461 waiting on condition [0x00007fc10dd7c000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007fc1dc07a458> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) at org.apache.ranger.audit.queue.AuditFileSpool.runLogAudit(AuditFileSpool.java:779) at org.apache.ranger.audit.queue.AuditFileSpool.run(AuditFileSpool.java:757) at java.lang.Thread.run(Thread.java:750) "Connection evictor" #232 daemon prio=5 os_prio=0 tid=0x00007fc97c6be800 nid=0x7460 waiting on condition [0x00007fc10de7d000] java.lang.Thread.State: TIMED_WAITING (sleeping) at java.lang.Thread.sleep(Native Method) at org.apache.http.impl.client.IdleConnectionEvictor$1.run(IdleConnectionEvictor.java:66) at java.lang.Thread.run(Thread.java:750) "EventStorePublisherWithRetry-0" #228 daemon prio=5 os_prio=0 tid=0x00007fc9848d6000 nid=0x745c waiting on condition [0x00007fc10e481000] java.lang.Thread.State: WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007fc1c9da8cc8> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:750) "Idle-Rpc-Conn-Sweeper-pool-0" #226 daemon prio=5 os_prio=0 tid=0x00007fc97c412800 nid=0x745a waiting on condition [0x00007fc10e782000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007fc1dc08d410> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1093) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:750) "ReadOnlyZKClient-cod-1e3xsc06g8mjm-master1.exait3p.z1did1.b0.cloudera.site:2181,cod-1e3xsc06g8mjm-master0.exait3p.z1did1.b0.cloudera.site:2181,cod-1e3xsc06g8mjm-leader0.exait3p.z1did1.b0.cloudera.site:2181@0x03b30e4c" #221 daemon prio=5 os_prio=0 tid=0x00007fc97c33b800 nid=0x7456 waiting on condition [0x00007fc10eb86000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007fc1dc079c60> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.DelayQueue.poll(DelayQueue.java:259) at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:323) at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$$Lambda$164/1053864765.run(Unknown Source) at java.lang.Thread.run(Thread.java:750) "AsyncConn Chore Service.Chore.1" #220 daemon prio=5 os_prio=0 tid=0x00007fc97c340800 nid=0x7454 waiting on condition [0x00007fc10ec87000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007fc1dc08d650> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1093) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:750) "regionserver/cod-1e3xsc06g8mjm-worker124:16020" #36 prio=5 os_prio=0 tid=0x00007fc11c64d800 nid=0x7453 in Object.wait() [0x00007fc10ef88000] java.lang.Thread.State: TIMED_WAITING (on object monitor) at java.lang.Object.wait(Native Method) at org.apache.hadoop.hbase.util.Sleeper.sleep(Sleeper.java:84) - locked <0x00007fc1c9db64a8> (a java.lang.Object) at org.apache.hadoop.hbase.util.Sleeper.sleep(Sleeper.java:67) at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:1095) "Session-HouseKeeper-4e6489a4-1" #219 prio=5 os_prio=0 tid=0x00007fc11da92000 nid=0x7452 waiting on condition [0x00007fc10f089000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007fc1c9fe1eb0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1093) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:750) "qtp1614356410-218" #218 daemon prio=5 os_prio=0 tid=0x00007fc11da49000 nid=0x7451 waiting on condition [0x00007fc10f38a000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007fc1c9fe20f0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at org.apache.hbase.thirdparty.org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:382) at org.apache.hbase.thirdparty.org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.idleJobPoll(QueuedThreadPool.java:974) at org.apache.hbase.thirdparty.org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:1018) at java.lang.Thread.run(Thread.java:750) "qtp1614356410-217" #217 daemon prio=5 os_prio=0 tid=0x00007fc11da47000 nid=0x7450 waiting on condition [0x00007fc10f48b000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007fc1c9fe25b0> (a java.util.concurrent.SynchronousQueue$TransferStack) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460) at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362) at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941) at org.apache.hbase.thirdparty.org.eclipse.jetty.util.thread.ReservedThreadExecutor$ReservedThread.reservedWait(ReservedThreadExecutor.java:324) at org.apache.hbase.thirdparty.org.eclipse.jetty.util.thread.ReservedThreadExecutor$ReservedThread.run(ReservedThreadExecutor.java:399) at org.apache.hbase.thirdparty.org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:883) at org.apache.hbase.thirdparty.org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:1034) at java.lang.Thread.run(Thread.java:750) "qtp1614356410-216" #216 daemon prio=5 os_prio=0 tid=0x00007fc11da45000 nid=0x744f waiting on condition [0x00007fc10f58c000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007fc1c9fe20f0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at org.apache.hbase.thirdparty.org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:382) at org.apache.hbase.thirdparty.org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.idleJobPoll(QueuedThreadPool.java:974) at org.apache.hbase.thirdparty.org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:1018) at java.lang.Thread.run(Thread.java:750) "qtp1614356410-215" #215 daemon prio=5 os_prio=0 tid=0x00007fc11da43800 nid=0x744e runnable [0x00007fc10f68d000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007fc1dc079d78> (a sun.nio.ch.Util$3) - locked <0x00007fc1dc079d88> (a java.util.Collections$UnmodifiableSet) - locked <0x00007fc1dc079d30> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:101) at org.apache.hbase.thirdparty.org.eclipse.jetty.io.ManagedSelector.nioSelect(ManagedSelector.java:183) at org.apache.hbase.thirdparty.org.eclipse.jetty.io.ManagedSelector.select(ManagedSelector.java:190) at org.apache.hbase.thirdparty.org.eclipse.jetty.io.ManagedSelector$SelectorProducer.select(ManagedSelector.java:606) at org.apache.hbase.thirdparty.org.eclipse.jetty.io.ManagedSelector$SelectorProducer.produce(ManagedSelector.java:543) at org.apache.hbase.thirdparty.org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produceTask(EatWhatYouKill.java:362) at org.apache.hbase.thirdparty.org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.doProduce(EatWhatYouKill.java:186) at org.apache.hbase.thirdparty.org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.tryProduce(EatWhatYouKill.java:173) at org.apache.hbase.thirdparty.org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.run(EatWhatYouKill.java:131) at org.apache.hbase.thirdparty.org.eclipse.jetty.util.thread.ReservedThreadExecutor$ReservedThread.run(ReservedThreadExecutor.java:409) at org.apache.hbase.thirdparty.org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:883) at org.apache.hbase.thirdparty.org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:1034) at java.lang.Thread.run(Thread.java:750) "qtp1614356410-214" #214 daemon prio=5 os_prio=0 tid=0x00007fc11da41800 nid=0x744d waiting on condition [0x00007fc10f78e000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007fc1c9fe20f0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at org.apache.hbase.thirdparty.org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:382) at org.apache.hbase.thirdparty.org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.idleJobPoll(QueuedThreadPool.java:974) at org.apache.hbase.thirdparty.org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:1018) at java.lang.Thread.run(Thread.java:750) "qtp1614356410-213" #213 daemon prio=5 os_prio=0 tid=0x00007fc11da3f800 nid=0x744c waiting on condition [0x00007fc10f88f000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007fc1c9fe20f0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at org.apache.hbase.thirdparty.org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:382) at org.apache.hbase.thirdparty.org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.idleJobPoll(QueuedThreadPool.java:974) at org.apache.hbase.thirdparty.org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:1018) at java.lang.Thread.run(Thread.java:750) "qtp1614356410-212-acceptor-0@43e04de3-ServerConnector@690eda7a{SSL, (ssl, http/1.1)}{0.0.0.0:16030}" #212 daemon prio=3 os_prio=0 tid=0x00007fc11da3d800 nid=0x744b runnable [0x00007fc10f990000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.ServerSocketChannelImpl.accept0(Native Method) at sun.nio.ch.ServerSocketChannelImpl.accept(ServerSocketChannelImpl.java:421) at sun.nio.ch.ServerSocketChannelImpl.accept(ServerSocketChannelImpl.java:249) - locked <0x00007fc1dc07a888> (a java.lang.Object) at org.apache.hbase.thirdparty.org.eclipse.jetty.server.ServerConnector.accept(ServerConnector.java:388) at org.apache.hbase.thirdparty.org.eclipse.jetty.server.AbstractConnector$Acceptor.run(AbstractConnector.java:702) at org.apache.hbase.thirdparty.org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:883) at org.apache.hbase.thirdparty.org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:1034) at java.lang.Thread.run(Thread.java:750) "qtp1614356410-211" #211 daemon prio=5 os_prio=0 tid=0x00007fc11da3c000 nid=0x744a waiting on condition [0x00007fc10fa91000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007fc1c9fe20f0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at org.apache.hbase.thirdparty.org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:382) at org.apache.hbase.thirdparty.org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.idleJobPoll(QueuedThreadPool.java:974) at org.apache.hbase.thirdparty.org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:1018) at java.lang.Thread.run(Thread.java:750) "RpcServer.metaPriority.FPBQ.Fifo.handler=0,queue=0,port=16020" #210 daemon prio=5 os_prio=0 tid=0x00007fc11d8f0800 nid=0x7447 waiting on condition [0x00007fc10ff92000] java.lang.Thread.State: WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007fc1c9fe8098> (a java.util.concurrent.Semaphore$NonfairSync) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer.parkAndCheckInterrupt(AbstractQueuedSynchronizer.java:836) at java.util.concurrent.locks.AbstractQueuedSynchronizer.doAcquireSharedInterruptibly(AbstractQueuedSynchronizer.java:997) at java.util.concurrent.locks.AbstractQueuedSynchronizer.acquireSharedInterruptibly(AbstractQueuedSynchronizer.java:1304) at java.util.concurrent.Semaphore.acquire(Semaphore.java:312) at org.apache.hadoop.hbase.ipc.FastPathBalancedQueueRpcExecutor$FastPathHandler.getCallRunner(FastPathBalancedQueueRpcExecutor.java:109) at org.apache.hadoop.hbase.ipc.RpcExecutor$Handler.run(RpcExecutor.java:339) "RpcServer.replication.FPBQ.Fifo.handler=2,queue=0,port=16020" #209 daemon prio=5 os_prio=0 tid=0x00007fc11d8ee000 nid=0x7446 waiting on condition [0x00007fc110093000] java.lang.Thread.State: WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007fc1c9fe27d8> (a java.util.concurrent.Semaphore$NonfairSync) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer.parkAndCheckInterrupt(AbstractQueuedSynchronizer.java:836) at java.util.concurrent.locks.AbstractQueuedSynchronizer.doAcquireSharedInterruptibly(AbstractQueuedSynchronizer.java:997) at java.util.concurrent.locks.AbstractQueuedSynchronizer.acquireSharedInterruptibly(AbstractQueuedSynchronizer.java:1304) at java.util.concurrent.Semaphore.acquire(Semaphore.java:312) at org.apache.hadoop.hbase.ipc.FastPathBalancedQueueRpcExecutor$FastPathHandler.getCallRunner(FastPathBalancedQueueRpcExecutor.java:109) at org.apache.hadoop.hbase.ipc.RpcExecutor$Handler.run(RpcExecutor.java:339) "RpcServer.replication.FPBQ.Fifo.handler=1,queue=0,port=16020" #208 daemon prio=5 os_prio=0 tid=0x00007fc11d8ec000 nid=0x7445 waiting on condition [0x00007fc110194000] java.lang.Thread.State: WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007fc1c9fe4160> (a java.util.concurrent.Semaphore$NonfairSync) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer.parkAndCheckInterrupt(AbstractQueuedSynchronizer.java:836) at java.util.concurrent.locks.AbstractQueuedSynchronizer.doAcquireSharedInterruptibly(AbstractQueuedSynchronizer.java:997) at java.util.concurrent.locks.AbstractQueuedSynchronizer.acquireSharedInterruptibly(AbstractQueuedSynchronizer.java:1304) at java.util.concurrent.Semaphore.acquire(Semaphore.java:312) at org.apache.hadoop.hbase.ipc.FastPathBalancedQueueRpcExecutor$FastPathHandler.getCallRunner(FastPathBalancedQueueRpcExecutor.java:109) at org.apache.hadoop.hbase.ipc.RpcExecutor$Handler.run(RpcExecutor.java:339) "RpcServer.replication.FPBQ.Fifo.handler=0,queue=0,port=16020" #207 daemon prio=5 os_prio=0 tid=0x00007fc11d8ea000 nid=0x7444 waiting on condition [0x00007fc110295000] java.lang.Thread.State: WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007fc1c9fe6048> (a java.util.concurrent.Semaphore$NonfairSync) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer.parkAndCheckInterrupt(AbstractQueuedSynchronizer.java:836) at java.util.concurrent.locks.AbstractQueuedSynchronizer.doAcquireSharedInterruptibly(AbstractQueuedSynchronizer.java:997) at java.util.concurrent.locks.AbstractQueuedSynchronizer.acquireSharedInterruptibly(AbstractQueuedSynchronizer.java:1304) at java.util.concurrent.Semaphore.acquire(Semaphore.java:312) at org.apache.hadoop.hbase.ipc.FastPathBalancedQueueRpcExecutor$FastPathHandler.getCallRunner(FastPathBalancedQueueRpcExecutor.java:109) at org.apache.hadoop.hbase.ipc.RpcExecutor$Handler.run(RpcExecutor.java:339) "RpcServer.priority.RWQ.Fifo.read.handler=19,queue=1,port=16020" #206 daemon prio=5 os_prio=0 tid=0x00007fc11d8e8000 nid=0x7443 waiting on condition [0x00007fc110396000] java.lang.Thread.State: WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007fc1bc116828> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) at org.apache.hadoop.hbase.ipc.RpcExecutor$Handler.getCallRunner(RpcExecutor.java:330) at org.apache.hadoop.hbase.ipc.RpcExecutor$Handler.run(RpcExecutor.java:339) "RpcServer.priority.RWQ.Fifo.read.handler=18,queue=1,port=16020" #205 daemon prio=5 os_prio=0 tid=0x00007fc11d8e5800 nid=0x7442 waiting on condition [0x00007fc110497000] java.lang.Thread.State: WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007fc1bc116828> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) at org.apache.hadoop.hbase.ipc.RpcExecutor$Handler.getCallRunner(RpcExecutor.java:330) at org.apache.hadoop.hbase.ipc.RpcExecutor$Handler.run(RpcExecutor.java:339) "RpcServer.priority.RWQ.Fifo.read.handler=17,queue=1,port=16020" #204 daemon prio=5 os_prio=0 tid=0x00007fc11d8e3800 nid=0x7441 waiting on condition [0x00007fc110598000] java.lang.Thread.State: WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007fc1bc116828> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) at org.apache.hadoop.hbase.ipc.RpcExecutor$Handler.getCallRunner(RpcExecutor.java:330) at org.apache.hadoop.hbase.ipc.RpcExecutor$Handler.run(RpcExecutor.java:339) "RpcServer.priority.RWQ.Fifo.read.handler=16,queue=1,port=16020" #203 daemon prio=5 os_prio=0 tid=0x00007fc11d8e1800 nid=0x7440 waiting on condition [0x00007fc110699000] java.lang.Thread.State: WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007fc1bc116828> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) at org.apache.hadoop.hbase.ipc.RpcExecutor$Handler.getCallRunner(RpcExecutor.java:330) at org.apache.hadoop.hbase.ipc.RpcExecutor$Handler.run(RpcExecutor.java:339) "RpcServer.priority.RWQ.Fifo.read.handler=15,queue=1,port=16020" #202 daemon prio=5 os_prio=0 tid=0x00007fc11d8df800 nid=0x743f waiting on condition [0x00007fc11079a000] java.lang.Thread.State: WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007fc1bc116828> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) at org.apache.hadoop.hbase.ipc.RpcExecutor$Handler.getCallRunner(RpcExecutor.java:330) at org.apache.hadoop.hbase.ipc.RpcExecutor$Handler.run(RpcExecutor.java:339) "RpcServer.priority.RWQ.Fifo.read.handler=14,queue=1,port=16020" #201 daemon prio=5 os_prio=0 tid=0x00007fc11d8dd800 nid=0x743e waiting on condition [0x00007fc11089b000] java.lang.Thread.State: WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007fc1bc116828> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) at org.apache.hadoop.hbase.ipc.RpcExecutor$Handler.getCallRunner(RpcExecutor.java:330) at org.apache.hadoop.hbase.ipc.RpcExecutor$Handler.run(RpcExecutor.java:339) "RpcServer.priority.RWQ.Fifo.read.handler=13,queue=1,port=16020" #200 daemon prio=5 os_prio=0 tid=0x00007fc11d8db000 nid=0x743d waiting on condition [0x00007fc11099c000] java.lang.Thread.State: WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007fc1bc116828> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) at org.apache.hadoop.hbase.ipc.RpcExecutor$Handler.getCallRunner(RpcExecutor.java:330) at org.apache.hadoop.hbase.ipc.RpcExecutor$Handler.run(RpcExecutor.java:339) "RpcServer.priority.RWQ.Fifo.read.handler=12,queue=1,port=16020" #199 daemon prio=5 os_prio=0 tid=0x00007fc11d8d9000 nid=0x743c waiting on condition [0x00007fc110a9d000] java.lang.Thread.State: WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007fc1bc116828> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) at org.apache.hadoop.hbase.ipc.RpcExecutor$Handler.getCallRunner(RpcExecutor.java:330) at org.apache.hadoop.hbase.ipc.RpcExecutor$Handler.run(RpcExecutor.java:339) "RpcServer.priority.RWQ.Fifo.read.handler=11,queue=1,port=16020" #198 daemon prio=5 os_prio=0 tid=0x00007fc11d8d6800 nid=0x743b waiting on condition [0x00007fc110b9e000] java.lang.Thread.State: WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007fc1bc116828> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) at org.apache.hadoop.hbase.ipc.RpcExecutor$Handler.getCallRunner(RpcExecutor.java:330) at org.apache.hadoop.hbase.ipc.RpcExecutor$Handler.run(RpcExecutor.java:339) "RpcServer.priority.RWQ.Fifo.read.handler=10,queue=1,port=16020" #197 daemon prio=5 os_prio=0 tid=0x00007fc11d8d4800 nid=0x743a waiting on condition [0x00007fc110c9f000] java.lang.Thread.State: WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007fc1bc116828> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) at org.apache.hadoop.hbase.ipc.RpcExecutor$Handler.getCallRunner(RpcExecutor.java:330) at org.apache.hadoop.hbase.ipc.RpcExecutor$Handler.run(RpcExecutor.java:339) "RpcServer.priority.RWQ.Fifo.read.handler=9,queue=1,port=16020" #196 daemon prio=5 os_prio=0 tid=0x00007fc11d8d2800 nid=0x7439 waiting on condition [0x00007fc110da0000] java.lang.Thread.State: WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007fc1bc116828> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) at org.apache.hadoop.hbase.ipc.RpcExecutor$Handler.getCallRunner(RpcExecutor.java:330) at org.apache.hadoop.hbase.ipc.RpcExecutor$Handler.run(RpcExecutor.java:339) "RpcServer.priority.RWQ.Fifo.read.handler=8,queue=1,port=16020" #195 daemon prio=5 os_prio=0 tid=0x00007fc11d8d0000 nid=0x7438 waiting on condition [0x00007fc110ea1000] java.lang.Thread.State: WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007fc1bc116828> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) at org.apache.hadoop.hbase.ipc.RpcExecutor$Handler.getCallRunner(RpcExecutor.java:330) at org.apache.hadoop.hbase.ipc.RpcExecutor$Handler.run(RpcExecutor.java:339) "RpcServer.priority.RWQ.Fifo.read.handler=7,queue=1,port=16020" #194 daemon prio=5 os_prio=0 tid=0x00007fc11d8ce000 nid=0x7437 waiting on condition [0x00007fc110fa2000] java.lang.Thread.State: WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007fc1bc116828> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) at org.apache.hadoop.hbase.ipc.RpcExecutor$Handler.getCallRunner(RpcExecutor.java:330) at org.apache.hadoop.hbase.ipc.RpcExecutor$Handler.run(RpcExecutor.java:339) "RpcServer.priority.RWQ.Fifo.read.handler=6,queue=1,port=16020" #193 daemon prio=5 os_prio=0 tid=0x00007fc11d8cc000 nid=0x7436 waiting on condition [0x00007fc1110a3000] java.lang.Thread.State: WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007fc1bc116828> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) at org.apache.hadoop.hbase.ipc.RpcExecutor$Handler.getCallRunner(RpcExecutor.java:330) at org.apache.hadoop.hbase.ipc.RpcExecutor$Handler.run(RpcExecutor.java:339) "RpcServer.priority.RWQ.Fifo.read.handler=5,queue=1,port=16020" #192 daemon prio=5 os_prio=0 tid=0x00007fc11d8ca000 nid=0x7435 waiting on condition [0x00007fc1111a4000] java.lang.Thread.State: WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007fc1bc116828> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) at org.apache.hadoop.hbase.ipc.RpcExecutor$Handler.getCallRunner(RpcExecutor.java:330) at org.apache.hadoop.hbase.ipc.RpcExecutor$Handler.run(RpcExecutor.java:339) "RpcServer.priority.RWQ.Fifo.read.handler=4,queue=1,port=16020" #191 daemon prio=5 os_prio=0 tid=0x00007fc11d8c7800 nid=0x7434 waiting on condition [0x00007fc1112a5000] java.lang.Thread.State: WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007fc1bc116828> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) at org.apache.hadoop.hbase.ipc.RpcExecutor$Handler.getCallRunner(RpcExecutor.java:330) at org.apache.hadoop.hbase.ipc.RpcExecutor$Handler.run(RpcExecutor.java:339) "RpcServer.priority.RWQ.Fifo.read.handler=3,queue=1,port=16020" #190 daemon prio=5 os_prio=0 tid=0x00007fc11d8c5800 nid=0x7433 waiting on condition [0x00007fc1113a6000] java.lang.Thread.State: WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007fc1bc116828> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) at org.apache.hadoop.hbase.ipc.RpcExecutor$Handler.getCallRunner(RpcExecutor.java:330) at org.apache.hadoop.hbase.ipc.RpcExecutor$Handler.run(RpcExecutor.java:339) "RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=16020" #189 daemon prio=5 os_prio=0 tid=0x00007fc11d8c3800 nid=0x7432 waiting on condition [0x00007fc1114a7000] java.lang.Thread.State: WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007fc1bc116828> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) at org.apache.hadoop.hbase.ipc.RpcExecutor$Handler.getCallRunner(RpcExecutor.java:330) at org.apache.hadoop.hbase.ipc.RpcExecutor$Handler.run(RpcExecutor.java:339) "RpcServer.priority.RWQ.Fifo.write.handler=1,queue=0,port=16020" #188 daemon prio=5 os_prio=0 tid=0x00007fc11d8c1000 nid=0x7431 waiting on condition [0x00007fc1115a8000] java.lang.Thread.State: WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007fc1bc0a9a90> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) at org.apache.hadoop.hbase.ipc.RpcExecutor$Handler.getCallRunner(RpcExecutor.java:330) at org.apache.hadoop.hbase.ipc.RpcExecutor$Handler.run(RpcExecutor.java:339) "RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16020" #187 daemon prio=5 os_prio=0 tid=0x00007fc11d8bf000 nid=0x7430 waiting on condition [0x00007fc1116a9000] java.lang.Thread.State: WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007fc1bc0a9a90> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) at org.apache.hadoop.hbase.ipc.RpcExecutor$Handler.getCallRunner(RpcExecutor.java:330) at org.apache.hadoop.hbase.ipc.RpcExecutor$Handler.run(RpcExecutor.java:339) "RpcServer.default.FPBQ.Fifo.handler=99,queue=9,port=16020" #186 daemon prio=5 os_prio=0 tid=0x00007fc11d8bd000 nid=0x742f waiting for monitor entry [0x00007fc1117a9000] java.lang.Thread.State: BLOCKED (on object monitor) at org.apache.hadoop.fs.FSInputStream.read(FSInputStream.java:73) - waiting to lock <0x00007fc4c2769660> (a org.apache.hadoop.fs.s3a.S3AInputStream) at org.apache.hadoop.fs.FSDataInputStream.read(FSDataInputStream.java:97) at org.apache.hadoop.hbase.io.FileLink$FileLinkInputStream.read(FileLink.java:171) at org.apache.hadoop.fs.FSDataInputStream.read(FSDataInputStream.java:97) at org.apache.hadoop.hbase.io.util.BlockIOUtils.preadWithExtra(BlockIOUtils.java:235) at org.apache.hadoop.hbase.io.hfile.HFileBlock$FSReaderImpl.readAtOffset(HFileBlock.java:1564) at org.apache.hadoop.hbase.io.hfile.HFileBlock$FSReaderImpl.readBlockDataInternal(HFileBlock.java:1788) at org.apache.hadoop.hbase.io.hfile.HFileBlock$FSReaderImpl.readBlockData(HFileBlock.java:1597) at org.apache.hadoop.hbase.io.hfile.HFileReaderImpl.readBlock(HFileReaderImpl.java:1347) at org.apache.hadoop.hbase.io.hfile.HFileReaderImpl.readBlock(HFileReaderImpl.java:1268) at org.apache.hadoop.hbase.io.hfile.HFileReaderImpl$HFileScannerImpl.readNextDataBlock(HFileReaderImpl.java:762) at org.apache.hadoop.hbase.io.hfile.HFileReaderImpl$EncodedScanner.next(HFileReaderImpl.java:1540) at org.apache.hadoop.hbase.regionserver.StoreFileScanner.reseekAtOrAfter(StoreFileScanner.java:351) at org.apache.hadoop.hbase.regionserver.StoreFileScanner.reseek(StoreFileScanner.java:244) at org.apache.hadoop.hbase.regionserver.NonLazyKeyValueScanner.doRealSeek(NonLazyKeyValueScanner.java:55) at org.apache.hadoop.hbase.regionserver.KeyValueHeap.generalizedSeek(KeyValueHeap.java:322) at org.apache.hadoop.hbase.regionserver.KeyValueHeap.reseek(KeyValueHeap.java:265) at org.apache.hadoop.hbase.regionserver.StoreScanner.reseek(StoreScanner.java:1095) at org.apache.hadoop.hbase.regionserver.StoreScanner.seekAsDirection(StoreScanner.java:1084) at org.apache.hadoop.hbase.regionserver.StoreScanner.seekOrSkipToNextColumn(StoreScanner.java:821) at org.apache.hadoop.hbase.regionserver.StoreScanner.next(StoreScanner.java:684) at org.apache.hadoop.hbase.regionserver.KeyValueHeap.next(KeyValueHeap.java:155) at org.apache.hadoop.hbase.regionserver.HRegion$RegionScannerImpl.populateResult(HRegion.java:7399) at org.apache.hadoop.hbase.regionserver.HRegion$RegionScannerImpl.nextInternal(HRegion.java:7567) at org.apache.hadoop.hbase.regionserver.HRegion$RegionScannerImpl.nextRaw(HRegion.java:7331) at org.apache.hadoop.hbase.regionserver.HRegion$RegionScannerImpl.nextRaw(HRegion.java:7317) at org.apache.phoenix.coprocessor.PagedRegionScanner.next(PagedRegionScanner.java:57) at org.apache.phoenix.coprocessor.PagedRegionScanner.nextRaw(PagedRegionScanner.java:96) at org.apache.phoenix.iterate.RegionScannerFactory$1.nextRaw(RegionScannerFactory.java:211) at org.apache.phoenix.coprocessor.DelegateRegionScanner.nextRaw(DelegateRegionScanner.java:77) at org.apache.phoenix.coprocessor.DelegateRegionScanner.nextRaw(DelegateRegionScanner.java:77) at org.apache.phoenix.coprocessor.BaseScannerRegionObserver$RegionScannerHolder.nextRaw(BaseScannerRegionObserver.java:308) at org.apache.hadoop.hbase.regionserver.RSRpcServices.scan(RSRpcServices.java:3373) - locked <0x00007fc85f003480> (a org.apache.phoenix.coprocessor.BaseScannerRegionObserver$RegionScannerHolder) at org.apache.hadoop.hbase.regionserver.RSRpcServices.scan(RSRpcServices.java:3637) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45819) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:392) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:140) at org.apache.hadoop.hbase.ipc.RpcExecutor$Handler.run(RpcExecutor.java:359) at org.apache.hadoop.hbase.ipc.RpcExecutor$Handler.run(RpcExecutor.java:339) "RpcServer.default.FPBQ.Fifo.handler=98,queue=8,port=16020" #185 daemon prio=5 os_prio=0 tid=0x00007fc11d8bb000 nid=0x742e waiting for monitor entry [0x00007fc1118aa000] java.lang.Thread.State: BLOCKED (on object monitor) at org.apache.hadoop.fs.FSInputStream.read(FSInputStream.java:73) - waiting to lock <0x00007fc4c2769660> (a org.apache.hadoop.fs.s3a.S3AInputStream) at org.apache.hadoop.fs.FSDataInputStream.read(FSDataInputStream.java:97) at org.apache.hadoop.hbase.io.FileLink$FileLinkInputStream.read(FileLink.java:171) at org.apache.hadoop.fs.FSDataInputStream.read(FSDataInputStream.java:97) at org.apache.hadoop.hbase.io.util.BlockIOUtils.preadWithExtra(BlockIOUtils.java:235) at org.apache.hadoop.hbase.io.hfile.HFileBlock$FSReaderImpl.readAtOffset(HFileBlock.java:1564) at org.apache.hadoop.hbase.io.hfile.HFileBlock$FSReaderImpl.readBlockDataInternal(HFileBlock.java:1788) at org.apache.hadoop.hbase.io.hfile.HFileBlock$FSReaderImpl.readBlockData(HFileBlock.java:1597) at org.apache.hadoop.hbase.io.hfile.HFileReaderImpl.readBlock(HFileReaderImpl.java:1347) at org.apache.hadoop.hbase.io.hfile.HFileReaderImpl.readBlock(HFileReaderImpl.java:1268) at org.apache.hadoop.hbase.io.hfile.HFileReaderImpl$HFileScannerImpl.readNextDataBlock(HFileReaderImpl.java:762) at org.apache.hadoop.hbase.io.hfile.HFileReaderImpl$EncodedScanner.next(HFileReaderImpl.java:1540) at org.apache.hadoop.hbase.regionserver.StoreFileScanner.reseekAtOrAfter(StoreFileScanner.java:351) at org.apache.hadoop.hbase.regionserver.StoreFileScanner.reseek(StoreFileScanner.java:244) at org.apache.hadoop.hbase.regionserver.NonLazyKeyValueScanner.doRealSeek(NonLazyKeyValueScanner.java:55) at org.apache.hadoop.hbase.regionserver.KeyValueHeap.generalizedSeek(KeyValueHeap.java:322) at org.apache.hadoop.hbase.regionserver.KeyValueHeap.reseek(KeyValueHeap.java:265) at org.apache.hadoop.hbase.regionserver.StoreScanner.reseek(StoreScanner.java:1095) at org.apache.hadoop.hbase.regionserver.StoreScanner.seekAsDirection(StoreScanner.java:1084) at org.apache.hadoop.hbase.regionserver.StoreScanner.seekOrSkipToNextColumn(StoreScanner.java:821) at org.apache.hadoop.hbase.regionserver.StoreScanner.next(StoreScanner.java:684) at org.apache.hadoop.hbase.regionserver.KeyValueHeap.next(KeyValueHeap.java:155) at org.apache.hadoop.hbase.regionserver.HRegion$RegionScannerImpl.populateResult(HRegion.java:7399) at org.apache.hadoop.hbase.regionserver.HRegion$RegionScannerImpl.nextInternal(HRegion.java:7567) at org.apache.hadoop.hbase.regionserver.HRegion$RegionScannerImpl.nextRaw(HRegion.java:7331) at org.apache.hadoop.hbase.regionserver.HRegion$RegionScannerImpl.nextRaw(HRegion.java:7317) at org.apache.phoenix.coprocessor.PagedRegionScanner.next(PagedRegionScanner.java:57) at org.apache.phoenix.coprocessor.PagedRegionScanner.nextRaw(PagedRegionScanner.java:96) at org.apache.phoenix.iterate.RegionScannerFactory$1.nextRaw(RegionScannerFactory.java:211) at org.apache.phoenix.coprocessor.DelegateRegionScanner.nextRaw(DelegateRegionScanner.java:77) at org.apache.phoenix.coprocessor.DelegateRegionScanner.nextRaw(DelegateRegionScanner.java:77) at org.apache.phoenix.coprocessor.BaseScannerRegionObserver$RegionScannerHolder.nextRaw(BaseScannerRegionObserver.java:308) at org.apache.hadoop.hbase.regionserver.RSRpcServices.scan(RSRpcServices.java:3373) - locked <0x00007fc93b208a38> (a org.apache.phoenix.coprocessor.BaseScannerRegionObserver$RegionScannerHolder) at org.apache.hadoop.hbase.regionserver.RSRpcServices.scan(RSRpcServices.java:3637) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45819) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:392) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:140) at org.apache.hadoop.hbase.ipc.RpcExecutor$Handler.run(RpcExecutor.java:359) at org.apache.hadoop.hbase.ipc.RpcExecutor$Handler.run(RpcExecutor.java:339) "RpcServer.default.FPBQ.Fifo.handler=97,queue=7,port=16020" #184 daemon prio=5 os_prio=0 tid=0x00007fc11d8b8800 nid=0x742d waiting for monitor entry [0x00007fc1119ab000] java.lang.Thread.State: BLOCKED (on object monitor) at org.apache.hadoop.fs.FSInputStream.read(FSInputStream.java:73) - waiting to lock <0x00007fc4c2769660> (a org.apache.hadoop.fs.s3a.S3AInputStream) at org.apache.hadoop.fs.FSDataInputStream.read(FSDataInputStream.java:97) at org.apache.hadoop.hbase.io.FileLink$FileLinkInputStream.read(FileLink.java:171) at org.apache.hadoop.fs.FSDataInputStream.read(FSDataInputStream.java:97) at org.apache.hadoop.hbase.io.util.BlockIOUtils.preadWithExtra(BlockIOUtils.java:235) at org.apache.hadoop.hbase.io.hfile.HFileBlock$FSReaderImpl.readAtOffset(HFileBlock.java:1564) at org.apache.hadoop.hbase.io.hfile.HFileBlock$FSReaderImpl.readBlockDataInternal(HFileBlock.java:1788) at org.apache.hadoop.hbase.io.hfile.HFileBlock$FSReaderImpl.readBlockData(HFileBlock.java:1597) at org.apache.hadoop.hbase.io.hfile.HFileReaderImpl.readBlock(HFileReaderImpl.java:1347) at org.apache.hadoop.hbase.io.hfile.HFileReaderImpl.readBlock(HFileReaderImpl.java:1268) at org.apache.hadoop.hbase.io.hfile.HFileReaderImpl$HFileScannerImpl.readNextDataBlock(HFileReaderImpl.java:762) at org.apache.hadoop.hbase.io.hfile.HFileReaderImpl$EncodedScanner.next(HFileReaderImpl.java:1540) at org.apache.hadoop.hbase.regionserver.StoreFileScanner.reseekAtOrAfter(StoreFileScanner.java:351) at org.apache.hadoop.hbase.regionserver.StoreFileScanner.reseek(StoreFileScanner.java:244) at org.apache.hadoop.hbase.regionserver.NonLazyKeyValueScanner.doRealSeek(NonLazyKeyValueScanner.java:55) at org.apache.hadoop.hbase.regionserver.KeyValueHeap.generalizedSeek(KeyValueHeap.java:322) at org.apache.hadoop.hbase.regionserver.KeyValueHeap.reseek(KeyValueHeap.java:265) at org.apache.hadoop.hbase.regionserver.StoreScanner.reseek(StoreScanner.java:1095) at org.apache.hadoop.hbase.regionserver.StoreScanner.seekAsDirection(StoreScanner.java:1084) at org.apache.hadoop.hbase.regionserver.StoreScanner.seekOrSkipToNextColumn(StoreScanner.java:821) at org.apache.hadoop.hbase.regionserver.StoreScanner.next(StoreScanner.java:684) at org.apache.hadoop.hbase.regionserver.KeyValueHeap.next(KeyValueHeap.java:155) at org.apache.hadoop.hbase.regionserver.HRegion$RegionScannerImpl.populateResult(HRegion.java:7399) at org.apache.hadoop.hbase.regionserver.HRegion$RegionScannerImpl.nextInternal(HRegion.java:7567) at org.apache.hadoop.hbase.regionserver.HRegion$RegionScannerImpl.nextRaw(HRegion.java:7331) at org.apache.hadoop.hbase.regionserver.HRegion$RegionScannerImpl.nextRaw(HRegion.java:7317) at org.apache.phoenix.coprocessor.PagedRegionScanner.next(PagedRegionScanner.java:57) at org.apache.phoenix.coprocessor.PagedRegionScanner.nextRaw(PagedRegionScanner.java:96) at org.apache.phoenix.iterate.RegionScannerFactory$1.nextRaw(RegionScannerFactory.java:211) at org.apache.phoenix.coprocessor.DelegateRegionScanner.nextRaw(DelegateRegionScanner.java:77) at org.apache.phoenix.coprocessor.DelegateRegionScanner.nextRaw(DelegateRegionScanner.java:77) at org.apache.phoenix.coprocessor.BaseScannerRegionObserver$RegionScannerHolder.nextRaw(BaseScannerRegionObserver.java:308) at org.apache.hadoop.hbase.regionserver.RSRpcServices.scan(RSRpcServices.java:3373) - locked <0x00007fc85f0330b8> (a org.apache.phoenix.coprocessor.BaseScannerRegionObserver$RegionScannerHolder) at org.apache.hadoop.hbase.regionserver.RSRpcServices.scan(RSRpcServices.java:3637) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45819) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:392) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:140) at org.apache.hadoop.hbase.ipc.RpcExecutor$Handler.run(RpcExecutor.java:359) at org.apache.hadoop.hbase.ipc.RpcExecutor$Handler.run(RpcExecutor.java:339) "RpcServer.default.FPBQ.Fifo.handler=96,queue=6,port=16020" #183 daemon prio=5 os_prio=0 tid=0x00007fc11d8b6800 nid=0x742c waiting for monitor entry [0x00007fc111aac000] java.lang.Thread.State: BLOCKED (on object monitor) at org.apache.hadoop.fs.FSInputStream.read(FSInputStream.java:73) - waiting to lock <0x00007fc4c2769660> (a org.apache.hadoop.fs.s3a.S3AInputStream) at org.apache.hadoop.fs.FSDataInputStream.read(FSDataInputStream.java:97) at org.apache.hadoop.hbase.io.FileLink$FileLinkInputStream.read(FileLink.java:171) at org.apache.hadoop.fs.FSDataInputStream.read(FSDataInputStream.java:97) at org.apache.hadoop.hbase.io.util.BlockIOUtils.preadWithExtra(BlockIOUtils.java:235) at org.apache.hadoop.hbase.io.hfile.HFileBlock$FSReaderImpl.readAtOffset(HFileBlock.java:1564) at org.apache.hadoop.hbase.io.hfile.HFileBlock$FSReaderImpl.readBlockDataInternal(HFileBlock.java:1788) at org.apache.hadoop.hbase.io.hfile.HFileBlock$FSReaderImpl.readBlockData(HFileBlock.java:1597) at org.apache.hadoop.hbase.io.hfile.HFileReaderImpl.readBlock(HFileReaderImpl.java:1347) at org.apache.hadoop.hbase.io.hfile.HFileReaderImpl.readBlock(HFileReaderImpl.java:1268) at org.apache.hadoop.hbase.io.hfile.HFileReaderImpl$HFileScannerImpl.readNextDataBlock(HFileReaderImpl.java:762) at org.apache.hadoop.hbase.io.hfile.HFileReaderImpl$EncodedScanner.next(HFileReaderImpl.java:1540) at org.apache.hadoop.hbase.regionserver.StoreFileScanner.reseekAtOrAfter(StoreFileScanner.java:351) at org.apache.hadoop.hbase.regionserver.StoreFileScanner.reseek(StoreFileScanner.java:244) at org.apache.hadoop.hbase.regionserver.NonLazyKeyValueScanner.doRealSeek(NonLazyKeyValueScanner.java:55) at org.apache.hadoop.hbase.regionserver.KeyValueHeap.generalizedSeek(KeyValueHeap.java:322) at org.apache.hadoop.hbase.regionserver.KeyValueHeap.reseek(KeyValueHeap.java:265) at org.apache.hadoop.hbase.regionserver.StoreScanner.reseek(StoreScanner.java:1095) at org.apache.hadoop.hbase.regionserver.StoreScanner.seekAsDirection(StoreScanner.java:1084) at org.apache.hadoop.hbase.regionserver.StoreScanner.seekOrSkipToNextColumn(StoreScanner.java:821) at org.apache.hadoop.hbase.regionserver.StoreScanner.next(StoreScanner.java:684) at org.apache.hadoop.hbase.regionserver.KeyValueHeap.next(KeyValueHeap.java:155) at org.apache.hadoop.hbase.regionserver.HRegion$RegionScannerImpl.populateResult(HRegion.java:7399) at org.apache.hadoop.hbase.regionserver.HRegion$RegionScannerImpl.nextInternal(HRegion.java:7567) at org.apache.hadoop.hbase.regionserver.HRegion$RegionScannerImpl.nextRaw(HRegion.java:7331) at org.apache.hadoop.hbase.regionserver.HRegion$RegionScannerImpl.nextRaw(HRegion.java:7317) at org.apache.phoenix.coprocessor.PagedRegionScanner.next(PagedRegionScanner.java:57) at org.apache.phoenix.coprocessor.PagedRegionScanner.nextRaw(PagedRegionScanner.java:96) at org.apache.phoenix.iterate.RegionScannerFactory$1.nextRaw(RegionScannerFactory.java:211) at org.apache.phoenix.coprocessor.DelegateRegionScanner.nextRaw(DelegateRegionScanner.java:77) at org.apache.phoenix.coprocessor.DelegateRegionScanner.nextRaw(DelegateRegionScanner.java:77) at org.apache.phoenix.coprocessor.BaseScannerRegionObserver$RegionScannerHolder.nextRaw(BaseScannerRegionObserver.java:308) at org.apache.hadoop.hbase.regionserver.RSRpcServices.scan(RSRpcServices.java:3373) - locked <0x00007fc85f0234a0> (a org.apache.phoenix.coprocessor.BaseScannerRegionObserver$RegionScannerHolder) at org.apache.hadoop.hbase.regionserver.RSRpcServices.scan(RSRpcServices.java:3637) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45819) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:392) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:140) at org.apache.hadoop.hbase.ipc.RpcExecutor$Handler.run(RpcExecutor.java:359) at org.apache.hadoop.hbase.ipc.RpcExecutor$Handler.run(RpcExecutor.java:339) "RpcServer.default.FPBQ.Fifo.handler=95,queue=5,port=16020" #182 daemon prio=5 os_prio=0 tid=0x00007fc11d8b4800 nid=0x742b waiting for monitor entry [0x00007fc111bad000] java.lang.Thread.State: BLOCKED (on object monitor) at org.apache.hadoop.fs.FSInputStream.read(FSInputStream.java:73) - waiting to lock <0x00007fc4c2769660> (a org.apache.hadoop.fs.s3a.S3AInputStream) at org.apache.hadoop.fs.FSDataInputStream.read(FSDataInputStream.java:97) at org.apache.hadoop.hbase.io.FileLink$FileLinkInputStream.read(FileLink.java:171) at org.apache.hadoop.fs.FSDataInputStream.read(FSDataInputStream.java:97) at org.apache.hadoop.hbase.io.util.BlockIOUtils.preadWithExtra(BlockIOUtils.java:235) at org.apache.hadoop.hbase.io.hfile.HFileBlock$FSReaderImpl.readAtOffset(HFileBlock.java:1564) at org.apache.hadoop.hbase.io.hfile.HFileBlock$FSReaderImpl.readBlockDataInternal(HFileBlock.java:1788) at org.apache.hadoop.hbase.io.hfile.HFileBlock$FSReaderImpl.readBlockData(HFileBlock.java:1597) at org.apache.hadoop.hbase.io.hfile.HFileReaderImpl.readBlock(HFileReaderImpl.java:1347) at org.apache.hadoop.hbase.io.hfile.HFileReaderImpl.readBlock(HFileReaderImpl.java:1268) at org.apache.hadoop.hbase.io.hfile.HFileReaderImpl$HFileScannerImpl.readNextDataBlock(HFileReaderImpl.java:762) at org.apache.hadoop.hbase.io.hfile.HFileReaderImpl$EncodedScanner.next(HFileReaderImpl.java:1540) at org.apache.hadoop.hbase.regionserver.StoreFileScanner.reseekAtOrAfter(StoreFileScanner.java:351) at org.apache.hadoop.hbase.regionserver.StoreFileScanner.reseek(StoreFileScanner.java:244) at org.apache.hadoop.hbase.regionserver.NonLazyKeyValueScanner.doRealSeek(NonLazyKeyValueScanner.java:55) at org.apache.hadoop.hbase.regionserver.KeyValueHeap.generalizedSeek(KeyValueHeap.java:322) at org.apache.hadoop.hbase.regionserver.KeyValueHeap.reseek(KeyValueHeap.java:265) at org.apache.hadoop.hbase.regionserver.StoreScanner.reseek(StoreScanner.java:1095) at org.apache.hadoop.hbase.regionserver.StoreScanner.seekAsDirection(StoreScanner.java:1084) at org.apache.hadoop.hbase.regionserver.StoreScanner.seekOrSkipToNextColumn(StoreScanner.java:821) at org.apache.hadoop.hbase.regionserver.StoreScanner.next(StoreScanner.java:684) at org.apache.hadoop.hbase.regionserver.KeyValueHeap.next(KeyValueHeap.java:155) at org.apache.hadoop.hbase.regionserver.HRegion$RegionScannerImpl.populateResult(HRegion.java:7399) at org.apache.hadoop.hbase.regionserver.HRegion$RegionScannerImpl.nextInternal(HRegion.java:7567) at org.apache.hadoop.hbase.regionserver.HRegion$RegionScannerImpl.nextRaw(HRegion.java:7331) at org.apache.hadoop.hbase.regionserver.HRegion$RegionScannerImpl.nextRaw(HRegion.java:7317) at org.apache.phoenix.coprocessor.PagedRegionScanner.next(PagedRegionScanner.java:57) at org.apache.phoenix.coprocessor.PagedRegionScanner.nextRaw(PagedRegionScanner.java:96) at org.apache.phoenix.iterate.RegionScannerFactory$1.nextRaw(RegionScannerFactory.java:211) at org.apache.phoenix.coprocessor.DelegateRegionScanner.nextRaw(DelegateRegionScanner.java:77) at org.apache.phoenix.coprocessor.DelegateRegionScanner.nextRaw(DelegateRegionScanner.java:77) at org.apache.phoenix.coprocessor.BaseScannerRegionObserver$RegionScannerHolder.nextRaw(BaseScannerRegionObserver.java:308) at org.apache.hadoop.hbase.regionserver.RSRpcServices.scan(RSRpcServices.java:3373) - locked <0x00007fc85f02b160> (a org.apache.phoenix.coprocessor.BaseScannerRegionObserver$RegionScannerHolder) at org.apache.hadoop.hbase.regionserver.RSRpcServices.scan(RSRpcServices.java:3637) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45819) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:392) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:140) at org.apache.hadoop.hbase.ipc.RpcExecutor$Handler.run(RpcExecutor.java:359) at org.apache.hadoop.hbase.ipc.RpcExecutor$Handler.run(RpcExecutor.java:339) "RpcServer.default.FPBQ.Fifo.handler=94,queue=4,port=16020" #181 daemon prio=5 os_prio=0 tid=0x00007fc11d8b2800 nid=0x742a waiting for monitor entry [0x00007fc111cae000] java.lang.Thread.State: BLOCKED (on object monitor) at org.apache.hadoop.fs.FSInputStream.read(FSInputStream.java:73) - waiting to lock <0x00007fc4c2769660> (a org.apache.hadoop.fs.s3a.S3AInputStream) at org.apache.hadoop.fs.FSDataInputStream.read(FSDataInputStream.java:97) at org.apache.hadoop.hbase.io.FileLink$FileLinkInputStream.read(FileLink.java:171) at org.apache.hadoop.fs.FSDataInputStream.read(FSDataInputStream.java:97) at org.apache.hadoop.hbase.io.util.BlockIOUtils.preadWithExtra(BlockIOUtils.java:235) at org.apache.hadoop.hbase.io.hfile.HFileBlock$FSReaderImpl.readAtOffset(HFileBlock.java:1564) at org.apache.hadoop.hbase.io.hfile.HFileBlock$FSReaderImpl.readBlockDataInternal(HFileBlock.java:1788) at org.apache.hadoop.hbase.io.hfile.HFileBlock$FSReaderImpl.readBlockData(HFileBlock.java:1597) at org.apache.hadoop.hbase.io.hfile.HFileReaderImpl.readBlock(HFileReaderImpl.java:1347) at org.apache.hadoop.hbase.io.hfile.HFileReaderImpl.readBlock(HFileReaderImpl.java:1268) at org.apache.hadoop.hbase.io.hfile.HFileReaderImpl$HFileScannerImpl.readNextDataBlock(HFileReaderImpl.java:762) at org.apache.hadoop.hbase.io.hfile.HFileReaderImpl$EncodedScanner.next(HFileReaderImpl.java:1540) at org.apache.hadoop.hbase.regionserver.StoreFileScanner.reseekAtOrAfter(StoreFileScanner.java:351) at org.apache.hadoop.hbase.regionserver.StoreFileScanner.reseek(StoreFileScanner.java:244) at org.apache.hadoop.hbase.regionserver.NonLazyKeyValueScanner.doRealSeek(NonLazyKeyValueScanner.java:55) at org.apache.hadoop.hbase.regionserver.KeyValueHeap.generalizedSeek(KeyValueHeap.java:322) at org.apache.hadoop.hbase.regionserver.KeyValueHeap.reseek(KeyValueHeap.java:265) at org.apache.hadoop.hbase.regionserver.StoreScanner.reseek(StoreScanner.java:1095) at org.apache.hadoop.hbase.regionserver.StoreScanner.seekAsDirection(StoreScanner.java:1084) at org.apache.hadoop.hbase.regionserver.StoreScanner.seekOrSkipToNextColumn(StoreScanner.java:821) at org.apache.hadoop.hbase.regionserver.StoreScanner.next(StoreScanner.java:684) at org.apache.hadoop.hbase.regionserver.KeyValueHeap.next(KeyValueHeap.java:155) at org.apache.hadoop.hbase.regionserver.HRegion$RegionScannerImpl.populateResult(HRegion.java:7399) at org.apache.hadoop.hbase.regionserver.HRegion$RegionScannerImpl.nextInternal(HRegion.java:7567) at org.apache.hadoop.hbase.regionserver.HRegion$RegionScannerImpl.nextRaw(HRegion.java:7331) at org.apache.hadoop.hbase.regionserver.HRegion$RegionScannerImpl.nextRaw(HRegion.java:7317) at org.apache.phoenix.coprocessor.PagedRegionScanner.next(PagedRegionScanner.java:57) at org.apache.phoenix.coprocessor.PagedRegionScanner.nextRaw(PagedRegionScanner.java:96) at org.apache.phoenix.iterate.RegionScannerFactory$1.nextRaw(RegionScannerFactory.java:211) at org.apache.phoenix.coprocessor.DelegateRegionScanner.nextRaw(DelegateRegionScanner.java:77) at org.apache.phoenix.coprocessor.DelegateRegionScanner.nextRaw(DelegateRegionScanner.java:77) at org.apache.phoenix.coprocessor.BaseScannerRegionObserver$RegionScannerHolder.nextRaw(BaseScannerRegionObserver.java:308) at org.apache.hadoop.hbase.regionserver.RSRpcServices.scan(RSRpcServices.java:3373) - locked <0x00007fc6b139d560> (a org.apache.phoenix.coprocessor.BaseScannerRegionObserver$RegionScannerHolder) at org.apache.hadoop.hbase.regionserver.RSRpcServices.scan(RSRpcServices.java:3637) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45819) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:392) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:140) at org.apache.hadoop.hbase.ipc.RpcExecutor$Handler.run(RpcExecutor.java:359) at org.apache.hadoop.hbase.ipc.RpcExecutor$Handler.run(RpcExecutor.java:339) "RpcServer.default.FPBQ.Fifo.handler=93,queue=3,port=16020" #180 daemon prio=5 os_prio=0 tid=0x00007fc11d8b0800 nid=0x7429 waiting for monitor entry [0x00007fc111daf000] java.lang.Thread.State: BLOCKED (on object monitor) at org.apache.hadoop.fs.FSInputStream.read(FSInputStream.java:73) - waiting to lock <0x00007fc4c2769660> (a org.apache.hadoop.fs.s3a.S3AInputStream) at org.apache.hadoop.fs.FSDataInputStream.read(FSDataInputStream.java:97) at org.apache.hadoop.hbase.io.FileLink$FileLinkInputStream.read(FileLink.java:171) at org.apache.hadoop.fs.FSDataInputStream.read(FSDataInputStream.java:97) at org.apache.hadoop.hbase.io.util.BlockIOUtils.preadWithExtra(BlockIOUtils.java:235) at org.apache.hadoop.hbase.io.hfile.HFileBlock$FSReaderImpl.readAtOffset(HFileBlock.java:1564) at org.apache.hadoop.hbase.io.hfile.HFileBlock$FSReaderImpl.readBlockDataInternal(HFileBlock.java:1788) at org.apache.hadoop.hbase.io.hfile.HFileBlock$FSReaderImpl.readBlockData(HFileBlock.java:1597) at org.apache.hadoop.hbase.io.hfile.HFileReaderImpl.readBlock(HFileReaderImpl.java:1347) at org.apache.hadoop.hbase.io.hfile.HFileReaderImpl.readBlock(HFileReaderImpl.java:1268) at org.apache.hadoop.hbase.io.hfile.HFileReaderImpl$HFileScannerImpl.readNextDataBlock(HFileReaderImpl.java:762) at org.apache.hadoop.hbase.io.hfile.HFileReaderImpl$EncodedScanner.next(HFileReaderImpl.java:1540) at org.apache.hadoop.hbase.regionserver.StoreFileScanner.reseekAtOrAfter(StoreFileScanner.java:351) at org.apache.hadoop.hbase.regionserver.StoreFileScanner.reseek(StoreFileScanner.java:244) at org.apache.hadoop.hbase.regionserver.NonLazyKeyValueScanner.doRealSeek(NonLazyKeyValueScanner.java:55) at org.apache.hadoop.hbase.regionserver.KeyValueHeap.generalizedSeek(KeyValueHeap.java:322) at org.apache.hadoop.hbase.regionserver.KeyValueHeap.reseek(KeyValueHeap.java:265) at org.apache.hadoop.hbase.regionserver.StoreScanner.reseek(StoreScanner.java:1095) at org.apache.hadoop.hbase.regionserver.StoreScanner.seekAsDirection(StoreScanner.java:1084) at org.apache.hadoop.hbase.regionserver.StoreScanner.seekOrSkipToNextColumn(StoreScanner.java:821) at org.apache.hadoop.hbase.regionserver.StoreScanner.next(StoreScanner.java:684) at org.apache.hadoop.hbase.regionserver.KeyValueHeap.next(KeyValueHeap.java:155) at org.apache.hadoop.hbase.regionserver.HRegion$RegionScannerImpl.populateResult(HRegion.java:7399) at org.apache.hadoop.hbase.regionserver.HRegion$RegionScannerImpl.nextInternal(HRegion.java:7567) at org.apache.hadoop.hbase.regionserver.HRegion$RegionScannerImpl.nextRaw(HRegion.java:7331) at org.apache.hadoop.hbase.regionserver.HRegion$RegionScannerImpl.nextRaw(HRegion.java:7317) at org.apache.phoenix.coprocessor.PagedRegionScanner.next(PagedRegionScanner.java:57) at org.apache.phoenix.coprocessor.PagedRegionScanner.nextRaw(PagedRegionScanner.java:96) at org.apache.phoenix.iterate.RegionScannerFactory$1.nextRaw(RegionScannerFactory.java:211) at org.apache.phoenix.coprocessor.DelegateRegionScanner.nextRaw(DelegateRegionScanner.java:77) at org.apache.phoenix.coprocessor.DelegateRegionScanner.nextRaw(DelegateRegionScanner.java:77) at org.apache.phoenix.coprocessor.BaseScannerRegionObserver$RegionScannerHolder.nextRaw(BaseScannerRegionObserver.java:308) at org.apache.hadoop.hbase.regionserver.RSRpcServices.scan(RSRpcServices.java:3373) - locked <0x00007fc85f01bc50> (a org.apache.phoenix.coprocessor.BaseScannerRegionObserver$RegionScannerHolder) at org.apache.hadoop.hbase.regionserver.RSRpcServices.scan(RSRpcServices.java:3637) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45819) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:392) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:140) at org.apache.hadoop.hbase.ipc.RpcExecutor$Handler.run(RpcExecutor.java:359) at org.apache.hadoop.hbase.ipc.RpcExecutor$Handler.run(RpcExecutor.java:339) "RpcServer.default.FPBQ.Fifo.handler=92,queue=2,port=16020" #179 daemon prio=5 os_prio=0 tid=0x00007fc11d8ae000 nid=0x7428 waiting for monitor entry [0x00007fc111eb0000] java.lang.Thread.State: BLOCKED (on object monitor) at org.apache.hadoop.fs.FSInputStream.read(FSInputStream.java:73) - waiting to lock <0x00007fc4c2769660> (a org.apache.hadoop.fs.s3a.S3AInputStream) at org.apache.hadoop.fs.FSDataInputStream.read(FSDataInputStream.java:97) at org.apache.hadoop.hbase.io.FileLink$FileLinkInputStream.read(FileLink.java:171) at org.apache.hadoop.fs.FSDataInputStream.read(FSDataInputStream.java:97) at org.apache.hadoop.hbase.io.util.BlockIOUtils.preadWithExtra(BlockIOUtils.java:235) at org.apache.hadoop.hbase.io.hfile.HFileBlock$FSReaderImpl.readAtOffset(HFileBlock.java:1564) at org.apache.hadoop.hbase.io.hfile.HFileBlock$FSReaderImpl.readBlockDataInternal(HFileBlock.java:1788) at org.apache.hadoop.hbase.io.hfile.HFileBlock$FSReaderImpl.readBlockData(HFileBlock.java:1597) at org.apache.hadoop.hbase.io.hfile.HFileReaderImpl.readBlock(HFileReaderImpl.java:1347) at org.apache.hadoop.hbase.io.hfile.HFileReaderImpl.readBlock(HFileReaderImpl.java:1268) at org.apache.hadoop.hbase.io.hfile.HFileReaderImpl$HFileScannerImpl.readNextDataBlock(HFileReaderImpl.java:762) at org.apache.hadoop.hbase.io.hfile.HFileReaderImpl$EncodedScanner.next(HFileReaderImpl.java:1540) at org.apache.hadoop.hbase.regionserver.StoreFileScanner.reseekAtOrAfter(StoreFileScanner.java:351) at org.apache.hadoop.hbase.regionserver.StoreFileScanner.reseek(StoreFileScanner.java:244) at org.apache.hadoop.hbase.regionserver.NonLazyKeyValueScanner.doRealSeek(NonLazyKeyValueScanner.java:55) at org.apache.hadoop.hbase.regionserver.KeyValueHeap.generalizedSeek(KeyValueHeap.java:322) at org.apache.hadoop.hbase.regionserver.KeyValueHeap.reseek(KeyValueHeap.java:265) at org.apache.hadoop.hbase.regionserver.StoreScanner.reseek(StoreScanner.java:1095) at org.apache.hadoop.hbase.regionserver.StoreScanner.seekAsDirection(StoreScanner.java:1084) at org.apache.hadoop.hbase.regionserver.StoreScanner.seekOrSkipToNextColumn(StoreScanner.java:821) at org.apache.hadoop.hbase.regionserver.StoreScanner.next(StoreScanner.java:684) at org.apache.hadoop.hbase.regionserver.KeyValueHeap.next(KeyValueHeap.java:155) at org.apache.hadoop.hbase.regionserver.HRegion$RegionScannerImpl.populateResult(HRegion.java:7399) at org.apache.hadoop.hbase.regionserver.HRegion$RegionScannerImpl.nextInternal(HRegion.java:7567) at org.apache.hadoop.hbase.regionserver.HRegion$RegionScannerImpl.nextRaw(HRegion.java:7331) at org.apache.hadoop.hbase.regionserver.HRegion$RegionScannerImpl.nextRaw(HRegion.java:7317) at org.apache.phoenix.coprocessor.PagedRegionScanner.next(PagedRegionScanner.java:57) at org.apache.phoenix.coprocessor.PagedRegionScanner.nextRaw(PagedRegionScanner.java:96) at org.apache.phoenix.iterate.RegionScannerFactory$1.nextRaw(RegionScannerFactory.java:211) at org.apache.phoenix.coprocessor.DelegateRegionScanner.nextRaw(DelegateRegionScanner.java:77) at org.apache.phoenix.coprocessor.DelegateRegionScanner.nextRaw(DelegateRegionScanner.java:77) at org.apache.phoenix.coprocessor.BaseScannerRegionObserver$RegionScannerHolder.nextRaw(BaseScannerRegionObserver.java:308) at org.apache.hadoop.hbase.regionserver.RSRpcServices.scan(RSRpcServices.java:3373) - locked <0x00007fc85f0151f0> (a org.apache.phoenix.coprocessor.BaseScannerRegionObserver$RegionScannerHolder) at org.apache.hadoop.hbase.regionserver.RSRpcServices.scan(RSRpcServices.java:3637) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45819) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:392) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:140) at org.apache.hadoop.hbase.ipc.RpcExecutor$Handler.run(RpcExecutor.java:359) at org.apache.hadoop.hbase.ipc.RpcExecutor$Handler.run(RpcExecutor.java:339) "RpcServer.default.FPBQ.Fifo.handler=91,queue=1,port=16020" #178 daemon prio=5 os_prio=0 tid=0x00007fc11d8ac000 nid=0x7427 waiting for monitor entry [0x00007fc111fb1000] java.lang.Thread.State: BLOCKED (on object monitor) at org.apache.hadoop.fs.FSInputStream.read(FSInputStream.java:73) - waiting to lock <0x00007fc4c2769660> (a org.apache.hadoop.fs.s3a.S3AInputStream) at org.apache.hadoop.fs.FSDataInputStream.read(FSDataInputStream.java:97) at org.apache.hadoop.hbase.io.FileLink$FileLinkInputStream.read(FileLink.java:171) at org.apache.hadoop.fs.FSDataInputStream.read(FSDataInputStream.java:97) at org.apache.hadoop.hbase.io.util.BlockIOUtils.preadWithExtra(BlockIOUtils.java:235) at org.apache.hadoop.hbase.io.hfile.HFileBlock$FSReaderImpl.readAtOffset(HFileBlock.java:1564) at org.apache.hadoop.hbase.io.hfile.HFileBlock$FSReaderImpl.readBlockDataInternal(HFileBlock.java:1788) at org.apache.hadoop.hbase.io.hfile.HFileBlock$FSReaderImpl.readBlockData(HFileBlock.java:1597) at org.apache.hadoop.hbase.io.hfile.HFileReaderImpl.readBlock(HFileReaderImpl.java:1347) at org.apache.hadoop.hbase.io.hfile.HFileReaderImpl.readBlock(HFileReaderImpl.java:1268) at org.apache.hadoop.hbase.io.hfile.HFileReaderImpl$HFileScannerImpl.readNextDataBlock(HFileReaderImpl.java:762) at org.apache.hadoop.hbase.io.hfile.HFileReaderImpl$EncodedScanner.next(HFileReaderImpl.java:1540) at org.apache.hadoop.hbase.regionserver.StoreFileScanner.reseekAtOrAfter(StoreFileScanner.java:351) at org.apache.hadoop.hbase.regionserver.StoreFileScanner.reseek(StoreFileScanner.java:244) at org.apache.hadoop.hbase.regionserver.NonLazyKeyValueScanner.doRealSeek(NonLazyKeyValueScanner.java:55) at org.apache.hadoop.hbase.regionserver.KeyValueHeap.generalizedSeek(KeyValueHeap.java:322) at org.apache.hadoop.hbase.regionserver.KeyValueHeap.reseek(KeyValueHeap.java:265) at org.apache.hadoop.hbase.regionserver.StoreScanner.reseek(StoreScanner.java:1095) at org.apache.hadoop.hbase.regionserver.StoreScanner.seekAsDirection(StoreScanner.java:1084) at org.apache.hadoop.hbase.regionserver.StoreScanner.seekOrSkipToNextColumn(StoreScanner.java:821) at org.apache.hadoop.hbase.regionserver.StoreScanner.next(StoreScanner.java:684) at org.apache.hadoop.hbase.regionserver.KeyValueHeap.next(KeyValueHeap.java:155) at org.apache.hadoop.hbase.regionserver.HRegion$RegionScannerImpl.populateResult(HRegion.java:7399) at org.apache.hadoop.hbase.regionserver.HRegion$RegionScannerImpl.nextInternal(HRegion.java:7567) at org.apache.hadoop.hbase.regionserver.HRegion$RegionScannerImpl.nextRaw(HRegion.java:7331) at org.apache.hadoop.hbase.regionserver.HRegion$RegionScannerImpl.nextRaw(HRegion.java:7317) at org.apache.phoenix.coprocessor.PagedRegionScanner.next(PagedRegionScanner.java:57) at org.apache.phoenix.coprocessor.PagedRegionScanner.nextRaw(PagedRegionScanner.java:96) at org.apache.phoenix.iterate.RegionScannerFactory$1.nextRaw(RegionScannerFactory.java:211) at org.apache.phoenix.coprocessor.DelegateRegionScanner.nextRaw(DelegateRegionScanner.java:77) at org.apache.phoenix.coprocessor.DelegateRegionScanner.nextRaw(DelegateRegionScanner.java:77) at org.apache.phoenix.coprocessor.BaseScannerRegionObserver$RegionScannerHolder.nextRaw(BaseScannerRegionObserver.java:308) at org.apache.hadoop.hbase.regionserver.RSRpcServices.scan(RSRpcServices.java:3373) - locked <0x00007fc85f025530> (a org.apache.phoenix.coprocessor.BaseScannerRegionObserver$RegionScannerHolder) at org.apache.hadoop.hbase.regionserver.RSRpcServices.scan(RSRpcServices.java:3637) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45819) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:392) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:140) at org.apache.hadoop.hbase.ipc.RpcExecutor$Handler.run(RpcExecutor.java:359) at org.apache.hadoop.hbase.ipc.RpcExecutor$Handler.run(RpcExecutor.java:339) "RpcServer.default.FPBQ.Fifo.handler=90,queue=0,port=16020" #177 daemon prio=5 os_prio=0 tid=0x00007fc11d8aa000 nid=0x7426 waiting for monitor entry [0x00007fc1120b2000] java.lang.Thread.State: BLOCKED (on object monitor) at org.apache.hadoop.fs.FSInputStream.read(FSInputStream.java:73) - waiting to lock <0x00007fc4c2769660> (a org.apache.hadoop.fs.s3a.S3AInputStream) at org.apache.hadoop.fs.FSDataInputStream.read(FSDataInputStream.java:97) at org.apache.hadoop.hbase.io.FileLink$FileLinkInputStream.read(FileLink.java:171) at org.apache.hadoop.fs.FSDataInputStream.read(FSDataInputStream.java:97) at org.apache.hadoop.hbase.io.util.BlockIOUtils.preadWithExtra(BlockIOUtils.java:235) at org.apache.hadoop.hbase.io.hfile.HFileBlock$FSReaderImpl.readAtOffset(HFileBlock.java:1564) at org.apache.hadoop.hbase.io.hfile.HFileBlock$FSReaderImpl.readBlockDataInternal(HFileBlock.java:1788) at org.apache.hadoop.hbase.io.hfile.HFileBlock$FSReaderImpl.readBlockData(HFileBlock.java:1597) at org.apache.hadoop.hbase.io.hfile.HFileReaderImpl.readBlock(HFileReaderImpl.java:1347) at org.apache.hadoop.hbase.io.hfile.HFileReaderImpl.readBlock(HFileReaderImpl.java:1268) at org.apache.hadoop.hbase.io.hfile.HFileReaderImpl$HFileScannerImpl.readNextDataBlock(HFileReaderImpl.java:762) at org.apache.hadoop.hbase.io.hfile.HFileReaderImpl$EncodedScanner.next(HFileReaderImpl.java:1540) at org.apache.hadoop.hbase.regionserver.StoreFileScanner.reseekAtOrAfter(StoreFileScanner.java:351) at org.apache.hadoop.hbase.regionserver.StoreFileScanner.reseek(StoreFileScanner.java:244) at org.apache.hadoop.hbase.regionserver.NonLazyKeyValueScanner.doRealSeek(NonLazyKeyValueScanner.java:55) at org.apache.hadoop.hbase.regionserver.KeyValueHeap.generalizedSeek(KeyValueHeap.java:322) at org.apache.hadoop.hbase.regionserver.KeyValueHeap.reseek(KeyValueHeap.java:265) at org.apache.hadoop.hbase.regionserver.StoreScanner.reseek(StoreScanner.java:1095) at org.apache.hadoop.hbase.regionserver.StoreScanner.seekAsDirection(StoreScanner.java:1084) at org.apache.hadoop.hbase.regionserver.StoreScanner.seekOrSkipToNextColumn(StoreScanner.java:821) at org.apache.hadoop.hbase.regionserver.StoreScanner.next(StoreScanner.java:684) at org.apache.hadoop.hbase.regionserver.KeyValueHeap.next(KeyValueHeap.java:155) at org.apache.hadoop.hbase.regionserver.HRegion$RegionScannerImpl.populateResult(HRegion.java:7399) at org.apache.hadoop.hbase.regionserver.HRegion$RegionScannerImpl.nextInternal(HRegion.java:7567) at org.apache.hadoop.hbase.regionserver.HRegion$RegionScannerImpl.nextRaw(HRegion.java:7331) at org.apache.hadoop.hbase.regionserver.HRegion$RegionScannerImpl.nextRaw(HRegion.java:7317) at org.apache.phoenix.coprocessor.PagedRegionScanner.next(PagedRegionScanner.java:57) at org.apache.phoenix.coprocessor.PagedRegionScanner.nextRaw(PagedRegionScanner.java:96) at org.apache.phoenix.iterate.RegionScannerFactory$1.nextRaw(RegionScannerFactory.java:211) at org.apache.phoenix.coprocessor.DelegateRegionScanner.nextRaw(DelegateRegionScanner.java:77) at org.apache.phoenix.coprocessor.DelegateRegionScanner.nextRaw(DelegateRegionScanner.java:77) at org.apache.phoenix.coprocessor.BaseScannerRegionObserver$RegionScannerHolder.nextRaw(BaseScannerRegionObserver.java:308) at org.apache.hadoop.hbase.regionserver.RSRpcServices.scan(RSRpcServices.java:3373) - locked <0x00007fc85f0351f8> (a org.apache.phoenix.coprocessor.BaseScannerRegionObserver$RegionScannerHolder) at org.apache.hadoop.hbase.regionserver.RSRpcServices.scan(RSRpcServices.java:3637) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45819) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:392) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:140) at org.apache.hadoop.hbase.ipc.RpcExecutor$Handler.run(RpcExecutor.java:359) at org.apache.hadoop.hbase.ipc.RpcExecutor$Handler.run(RpcExecutor.java:339) "RpcServer.default.FPBQ.Fifo.handler=89,queue=9,port=16020" #176 daemon prio=5 os_prio=0 tid=0x00007fc11d8a7800 nid=0x7425 waiting for monitor entry [0x00007fc1121b3000] java.lang.Thread.State: BLOCKED (on object monitor) at org.apache.hadoop.fs.FSInputStream.read(FSInputStream.java:73) - waiting to lock <0x00007fc4c2769660> (a org.apache.hadoop.fs.s3a.S3AInputStream) at org.apache.hadoop.fs.FSDataInputStream.read(FSDataInputStream.java:97) at org.apache.hadoop.hbase.io.FileLink$FileLinkInputStream.read(FileLink.java:171) at org.apache.hadoop.fs.FSDataInputStream.read(FSDataInputStream.java:97) at org.apache.hadoop.hbase.io.util.BlockIOUtils.preadWithExtra(BlockIOUtils.java:235) at org.apache.hadoop.hbase.io.hfile.HFileBlock$FSReaderImpl.readAtOffset(HFileBlock.java:1564) at org.apache.hadoop.hbase.io.hfile.HFileBlock$FSReaderImpl.readBlockDataInternal(HFileBlock.java:1788) at org.apache.hadoop.hbase.io.hfile.HFileBlock$FSReaderImpl.readBlockData(HFileBlock.java:1597) at org.apache.hadoop.hbase.io.hfile.HFileReaderImpl.readBlock(HFileReaderImpl.java:1347) at org.apache.hadoop.hbase.io.hfile.HFileReaderImpl.readBlock(HFileReaderImpl.java:1268) at org.apache.hadoop.hbase.io.hfile.HFileReaderImpl$HFileScannerImpl.readNextDataBlock(HFileReaderImpl.java:762) at org.apache.hadoop.hbase.io.hfile.HFileReaderImpl$EncodedScanner.next(HFileReaderImpl.java:1540) at org.apache.hadoop.hbase.regionserver.StoreFileScanner.reseekAtOrAfter(StoreFileScanner.java:351) at org.apache.hadoop.hbase.regionserver.StoreFileScanner.reseek(StoreFileScanner.java:244) at org.apache.hadoop.hbase.regionserver.NonLazyKeyValueScanner.doRealSeek(NonLazyKeyValueScanner.java:55) at org.apache.hadoop.hbase.regionserver.KeyValueHeap.generalizedSeek(KeyValueHeap.java:322) at org.apache.hadoop.hbase.regionserver.KeyValueHeap.reseek(KeyValueHeap.java:265) at org.apache.hadoop.hbase.regionserver.StoreScanner.reseek(StoreScanner.java:1095) at org.apache.hadoop.hbase.regionserver.StoreScanner.seekAsDirection(StoreScanner.java:1084) at org.apache.hadoop.hbase.regionserver.StoreScanner.seekOrSkipToNextColumn(StoreScanner.java:821) at org.apache.hadoop.hbase.regionserver.StoreScanner.next(StoreScanner.java:684) at org.apache.hadoop.hbase.regionserver.KeyValueHeap.next(KeyValueHeap.java:155) at org.apache.hadoop.hbase.regionserver.HRegion$RegionScannerImpl.populateResult(HRegion.java:7399) at org.apache.hadoop.hbase.regionserver.HRegion$RegionScannerImpl.nextInternal(HRegion.java:7567) at org.apache.hadoop.hbase.regionserver.HRegion$RegionScannerImpl.nextRaw(HRegion.java:7331) at org.apache.hadoop.hbase.regionserver.HRegion$RegionScannerImpl.nextRaw(HRegion.java:7317) at org.apache.phoenix.coprocessor.PagedRegionScanner.next(PagedRegionScanner.java:57) at org.apache.phoenix.coprocessor.PagedRegionScanner.nextRaw(PagedRegionScanner.java:96) at org.apache.phoenix.iterate.RegionScannerFactory$1.nextRaw(RegionScannerFactory.java:211) at org.apache.phoenix.coprocessor.DelegateRegionScanner.nextRaw(DelegateRegionScanner.java:77) at org.apache.phoenix.coprocessor.DelegateRegionScanner.nextRaw(DelegateRegionScanner.java:77) at org.apache.phoenix.coprocessor.BaseScannerRegionObserver$RegionScannerHolder.nextRaw(BaseScannerRegionObserver.java:308) at org.apache.hadoop.hbase.regionserver.RSRpcServices.scan(RSRpcServices.java:3373) - locked <0x00007fc85f00ca30> (a org.apache.phoenix.coprocessor.BaseScannerRegionObserver$RegionScannerHolder) at org.apache.hadoop.hbase.regionserver.RSRpcServices.scan(RSRpcServices.java:3637) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45819) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:392) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:140) at org.apache.hadoop.hbase.ipc.RpcExecutor$Handler.run(RpcExecutor.java:359) at org.apache.hadoop.hbase.ipc.RpcExecutor$Handler.run(RpcExecutor.java:339) "RpcServer.default.FPBQ.Fifo.handler=88,queue=8,port=16020" #175 daemon prio=5 os_prio=0 tid=0x00007fc11d8a5800 nid=0x7424 waiting for monitor entry [0x00007fc1122b4000] java.lang.Thread.State: BLOCKED (on object monitor) at org.apache.hadoop.fs.FSInputStream.read(FSInputStream.java:73) - waiting to lock <0x00007fc4c2769660> (a org.apache.hadoop.fs.s3a.S3AInputStream) at org.apache.hadoop.fs.FSDataInputStream.read(FSDataInputStream.java:97) at org.apache.hadoop.hbase.io.FileLink$FileLinkInputStream.read(FileLink.java:171) at org.apache.hadoop.fs.FSDataInputStream.read(FSDataInputStream.java:97) at org.apache.hadoop.hbase.io.util.BlockIOUtils.preadWithExtra(BlockIOUtils.java:235) at org.apache.hadoop.hbase.io.hfile.HFileBlock$FSReaderImpl.readAtOffset(HFileBlock.java:1564) at org.apache.hadoop.hbase.io.hfile.HFileBlock$FSReaderImpl.readBlockDataInternal(HFileBlock.java:1788) at org.apache.hadoop.hbase.io.hfile.HFileBlock$FSReaderImpl.readBlockData(HFileBlock.java:1597) at org.apache.hadoop.hbase.io.hfile.HFileReaderImpl.readBlock(HFileReaderImpl.java:1347) at org.apache.hadoop.hbase.io.hfile.HFileReaderImpl.readBlock(HFileReaderImpl.java:1268) at org.apache.hadoop.hbase.io.hfile.HFileReaderImpl$HFileScannerImpl.readNextDataBlock(HFileReaderImpl.java:762) at org.apache.hadoop.hbase.io.hfile.HFileReaderImpl$EncodedScanner.next(HFileReaderImpl.java:1540) at org.apache.hadoop.hbase.regionserver.StoreFileScanner.next(StoreFileScanner.java:196) at org.apache.hadoop.hbase.regionserver.KeyValueHeap.next(KeyValueHeap.java:122) at org.apache.hadoop.hbase.regionserver.StoreScanner.trySkipToNextRow(StoreScanner.java:884) at org.apache.hadoop.hbase.regionserver.StoreScanner.seekOrSkipToNextRow(StoreScanner.java:812) at org.apache.hadoop.hbase.regionserver.StoreScanner.next(StoreScanner.java:720) at org.apache.hadoop.hbase.regionserver.KeyValueHeap.next(KeyValueHeap.java:155) at org.apache.hadoop.hbase.regionserver.HRegion$RegionScannerImpl.populateResult(HRegion.java:7399) at org.apache.hadoop.hbase.regionserver.HRegion$RegionScannerImpl.nextInternal(HRegion.java:7567) at org.apache.hadoop.hbase.regionserver.HRegion$RegionScannerImpl.nextRaw(HRegion.java:7331) at org.apache.hadoop.hbase.regionserver.HRegion$RegionScannerImpl.nextRaw(HRegion.java:7317) at org.apache.phoenix.coprocessor.PagedRegionScanner.next(PagedRegionScanner.java:57) at org.apache.phoenix.coprocessor.PagedRegionScanner.nextRaw(PagedRegionScanner.java:96) at org.apache.phoenix.iterate.RegionScannerFactory$1.nextRaw(RegionScannerFactory.java:211) at org.apache.phoenix.coprocessor.DelegateRegionScanner.nextRaw(DelegateRegionScanner.java:77) at org.apache.phoenix.coprocessor.DelegateRegionScanner.nextRaw(DelegateRegionScanner.java:77) at org.apache.phoenix.coprocessor.BaseScannerRegionObserver$RegionScannerHolder.nextRaw(BaseScannerRegionObserver.java:308) at org.apache.hadoop.hbase.regionserver.RSRpcServices.scan(RSRpcServices.java:3373) - locked <0x00007fc6b139dec8> (a org.apache.phoenix.coprocessor.BaseScannerRegionObserver$RegionScannerHolder) at org.apache.hadoop.hbase.regionserver.RSRpcServices.scan(RSRpcServices.java:3637) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45819) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:392) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:140) at org.apache.hadoop.hbase.ipc.RpcExecutor$Handler.run(RpcExecutor.java:359) at org.apache.hadoop.hbase.ipc.RpcExecutor$Handler.run(RpcExecutor.java:339) "RpcServer.default.FPBQ.Fifo.handler=87,queue=7,port=16020" #174 daemon prio=5 os_prio=0 tid=0x00007fc11d8a3000 nid=0x7423 waiting for monitor entry [0x00007fc1123b5000] java.lang.Thread.State: BLOCKED (on object monitor) at org.apache.hadoop.fs.FSInputStream.read(FSInputStream.java:73) - waiting to lock <0x00007fc4c2769660> (a org.apache.hadoop.fs.s3a.S3AInputStream) at org.apache.hadoop.fs.FSDataInputStream.read(FSDataInputStream.java:97) at org.apache.hadoop.hbase.io.FileLink$FileLinkInputStream.read(FileLink.java:171) at org.apache.hadoop.fs.FSDataInputStream.read(FSDataInputStream.java:97) at org.apache.hadoop.hbase.io.util.BlockIOUtils.preadWithExtra(BlockIOUtils.java:235) at org.apache.hadoop.hbase.io.hfile.HFileBlock$FSReaderImpl.readAtOffset(HFileBlock.java:1564) at org.apache.hadoop.hbase.io.hfile.HFileBlock$FSReaderImpl.readBlockDataInternal(HFileBlock.java:1788) at org.apache.hadoop.hbase.io.hfile.HFileBlock$FSReaderImpl.readBlockData(HFileBlock.java:1597) at org.apache.hadoop.hbase.io.hfile.HFileReaderImpl.readBlock(HFileReaderImpl.java:1347) at org.apache.hadoop.hbase.io.hfile.HFileReaderImpl.readBlock(HFileReaderImpl.java:1268) at org.apache.hadoop.hbase.io.hfile.HFileReaderImpl$HFileScannerImpl.readNextDataBlock(HFileReaderImpl.java:762) at org.apache.hadoop.hbase.io.hfile.HFileReaderImpl$EncodedScanner.next(HFileReaderImpl.java:1540) at org.apache.hadoop.hbase.regionserver.StoreFileScanner.reseekAtOrAfter(StoreFileScanner.java:351) at org.apache.hadoop.hbase.regionserver.StoreFileScanner.reseek(StoreFileScanner.java:244) at org.apache.hadoop.hbase.regionserver.NonLazyKeyValueScanner.doRealSeek(NonLazyKeyValueScanner.java:55) at org.apache.hadoop.hbase.regionserver.KeyValueHeap.generalizedSeek(KeyValueHeap.java:322) at org.apache.hadoop.hbase.regionserver.KeyValueHeap.reseek(KeyValueHeap.java:265) at org.apache.hadoop.hbase.regionserver.StoreScanner.reseek(StoreScanner.java:1095) at org.apache.hadoop.hbase.regionserver.StoreScanner.seekAsDirection(StoreScanner.java:1084) at org.apache.hadoop.hbase.regionserver.StoreScanner.seekOrSkipToNextColumn(StoreScanner.java:821) at org.apache.hadoop.hbase.regionserver.StoreScanner.next(StoreScanner.java:684) at org.apache.hadoop.hbase.regionserver.KeyValueHeap.next(KeyValueHeap.java:155) at org.apache.hadoop.hbase.regionserver.HRegion$RegionScannerImpl.populateResult(HRegion.java:7399) at org.apache.hadoop.hbase.regionserver.HRegion$RegionScannerImpl.nextInternal(HRegion.java:7567) at org.apache.hadoop.hbase.regionserver.HRegion$RegionScannerImpl.nextRaw(HRegion.java:7331) at org.apache.hadoop.hbase.regionserver.HRegion$RegionScannerImpl.nextRaw(HRegion.java:7317) at org.apache.phoenix.coprocessor.PagedRegionScanner.next(PagedRegionScanner.java:57) at org.apache.phoenix.coprocessor.PagedRegionScanner.nextRaw(PagedRegionScanner.java:96) at org.apache.phoenix.iterate.RegionScannerFactory$1.nextRaw(RegionScannerFactory.java:211) at org.apache.phoenix.coprocessor.DelegateRegionScanner.nextRaw(DelegateRegionScanner.java:77) at org.apache.phoenix.coprocessor.DelegateRegionScanner.nextRaw(DelegateRegionScanner.java:77) at org.apache.phoenix.coprocessor.BaseScannerRegionObserver$RegionScannerHolder.nextRaw(BaseScannerRegionObserver.java:308) at org.apache.hadoop.hbase.regionserver.RSRpcServices.scan(RSRpcServices.java:3373) - locked <0x00007fc85f02d268> (a org.apache.phoenix.coprocessor.BaseScannerRegionObserver$RegionScannerHolder) at org.apache.hadoop.hbase.regionserver.RSRpcServices.scan(RSRpcServices.java:3637) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45819) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:392) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:140) at org.apache.hadoop.hbase.ipc.RpcExecutor$Handler.run(RpcExecutor.java:359) at org.apache.hadoop.hbase.ipc.RpcExecutor$Handler.run(RpcExecutor.java:339) "RpcServer.default.FPBQ.Fifo.handler=86,queue=6,port=16020" #173 daemon prio=5 os_prio=0 tid=0x00007fc11d8a1000 nid=0x7422 waiting for monitor entry [0x00007fc1124b6000] java.lang.Thread.State: BLOCKED (on object monitor) at org.apache.hadoop.fs.FSInputStream.read(FSInputStream.java:73) - waiting to lock <0x00007fc4c2769660> (a org.apache.hadoop.fs.s3a.S3AInputStream) at org.apache.hadoop.fs.FSDataInputStream.read(FSDataInputStream.java:97) at org.apache.hadoop.hbase.io.FileLink$FileLinkInputStream.read(FileLink.java:171) at org.apache.hadoop.fs.FSDataInputStream.read(FSDataInputStream.java:97) at org.apache.hadoop.hbase.io.util.BlockIOUtils.preadWithExtra(BlockIOUtils.java:235) at org.apache.hadoop.hbase.io.hfile.HFileBlock$FSReaderImpl.readAtOffset(HFileBlock.java:1564) at org.apache.hadoop.hbase.io.hfile.HFileBlock$FSReaderImpl.readBlockDataInternal(HFileBlock.java:1788) at org.apache.hadoop.hbase.io.hfile.HFileBlock$FSReaderImpl.readBlockData(HFileBlock.java:1597) at org.apache.hadoop.hbase.io.hfile.HFileReaderImpl.readBlock(HFileReaderImpl.java:1347) at org.apache.hadoop.hbase.io.hfile.HFileReaderImpl.readBlock(HFileReaderImpl.java:1268) at org.apache.hadoop.hbase.io.hfile.HFileReaderImpl$HFileScannerImpl.readNextDataBlock(HFileReaderImpl.java:762) at org.apache.hadoop.hbase.io.hfile.HFileReaderImpl$EncodedScanner.next(HFileReaderImpl.java:1540) at org.apache.hadoop.hbase.regionserver.StoreFileScanner.reseekAtOrAfter(StoreFileScanner.java:351) at org.apache.hadoop.hbase.regionserver.StoreFileScanner.reseek(StoreFileScanner.java:244) at org.apache.hadoop.hbase.regionserver.NonLazyKeyValueScanner.doRealSeek(NonLazyKeyValueScanner.java:55) at org.apache.hadoop.hbase.regionserver.KeyValueHeap.generalizedSeek(KeyValueHeap.java:322) at org.apache.hadoop.hbase.regionserver.KeyValueHeap.reseek(KeyValueHeap.java:265) at org.apache.hadoop.hbase.regionserver.StoreScanner.reseek(StoreScanner.java:1095) at org.apache.hadoop.hbase.regionserver.StoreScanner.seekAsDirection(StoreScanner.java:1084) at org.apache.hadoop.hbase.regionserver.StoreScanner.seekOrSkipToNextColumn(StoreScanner.java:821) at org.apache.hadoop.hbase.regionserver.StoreScanner.next(StoreScanner.java:684) at org.apache.hadoop.hbase.regionserver.KeyValueHeap.next(KeyValueHeap.java:155) at org.apache.hadoop.hbase.regionserver.HRegion$RegionScannerImpl.populateResult(HRegion.java:7399) at org.apache.hadoop.hbase.regionserver.HRegion$RegionScannerImpl.nextInternal(HRegion.java:7567) at org.apache.hadoop.hbase.regionserver.HRegion$RegionScannerImpl.nextRaw(HRegion.java:7331) at org.apache.hadoop.hbase.regionserver.HRegion$RegionScannerImpl.nextRaw(HRegion.java:7317) at org.apache.phoenix.coprocessor.PagedRegionScanner.next(PagedRegionScanner.java:57) at org.apache.phoenix.coprocessor.PagedRegionScanner.nextRaw(PagedRegionScanner.java:96) at org.apache.phoenix.iterate.RegionScannerFactory$1.nextRaw(RegionScannerFactory.java:211) at org.apache.phoenix.coprocessor.DelegateRegionScanner.nextRaw(DelegateRegionScanner.java:77) at org.apache.phoenix.coprocessor.DelegateRegionScanner.nextRaw(DelegateRegionScanner.java:77) at org.apache.phoenix.coprocessor.BaseScannerRegionObserver$RegionScannerHolder.nextRaw(BaseScannerRegionObserver.java:308) at org.apache.hadoop.hbase.regionserver.RSRpcServices.scan(RSRpcServices.java:3373) - locked <0x00007fc85f005680> (a org.apache.phoenix.coprocessor.BaseScannerRegionObserver$RegionScannerHolder) at org.apache.hadoop.hbase.regionserver.RSRpcServices.scan(RSRpcServices.java:3637) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45819) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:392) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:140) at org.apache.hadoop.hbase.ipc.RpcExecutor$Handler.run(RpcExecutor.java:359) at org.apache.hadoop.hbase.ipc.RpcExecutor$Handler.run(RpcExecutor.java:339) "RpcServer.default.FPBQ.Fifo.handler=85,queue=5,port=16020" #172 daemon prio=5 os_prio=0 tid=0x00007fc11d89f000 nid=0x7421 waiting for monitor entry [0x00007fc1125b7000] java.lang.Thread.State: BLOCKED (on object monitor) at org.apache.hadoop.fs.FSInputStream.read(FSInputStream.java:73) - waiting to lock <0x00007fc4c2769660> (a org.apache.hadoop.fs.s3a.S3AInputStream) at org.apache.hadoop.fs.FSDataInputStream.read(FSDataInputStream.java:97) at org.apache.hadoop.hbase.io.FileLink$FileLinkInputStream.read(FileLink.java:171) at org.apache.hadoop.fs.FSDataInputStream.read(FSDataInputStream.java:97) at org.apache.hadoop.hbase.io.util.BlockIOUtils.preadWithExtra(BlockIOUtils.java:235) at org.apache.hadoop.hbase.io.hfile.HFileBlock$FSReaderImpl.readAtOffset(HFileBlock.java:1564) at org.apache.hadoop.hbase.io.hfile.HFileBlock$FSReaderImpl.readBlockDataInternal(HFileBlock.java:1788) at org.apache.hadoop.hbase.io.hfile.HFileBlock$FSReaderImpl.readBlockData(HFileBlock.java:1597) at org.apache.hadoop.hbase.io.hfile.HFileReaderImpl.readBlock(HFileReaderImpl.java:1347) at org.apache.hadoop.hbase.io.hfile.HFileReaderImpl.readBlock(HFileReaderImpl.java:1268) at org.apache.hadoop.hbase.io.hfile.HFileReaderImpl$HFileScannerImpl.readNextDataBlock(HFileReaderImpl.java:762) at org.apache.hadoop.hbase.io.hfile.HFileReaderImpl$EncodedScanner.next(HFileReaderImpl.java:1540) at org.apache.hadoop.hbase.regionserver.StoreFileScanner.reseekAtOrAfter(StoreFileScanner.java:351) at org.apache.hadoop.hbase.regionserver.StoreFileScanner.reseek(StoreFileScanner.java:244) at org.apache.hadoop.hbase.regionserver.NonLazyKeyValueScanner.doRealSeek(NonLazyKeyValueScanner.java:55) at org.apache.hadoop.hbase.regionserver.KeyValueHeap.generalizedSeek(KeyValueHeap.java:322) at org.apache.hadoop.hbase.regionserver.KeyValueHeap.reseek(KeyValueHeap.java:265) at org.apache.hadoop.hbase.regionserver.StoreScanner.reseek(StoreScanner.java:1095) at org.apache.hadoop.hbase.regionserver.StoreScanner.seekAsDirection(StoreScanner.java:1084) at org.apache.hadoop.hbase.regionserver.StoreScanner.seekOrSkipToNextColumn(StoreScanner.java:821) at org.apache.hadoop.hbase.regionserver.StoreScanner.next(StoreScanner.java:684) at org.apache.hadoop.hbase.regionserver.KeyValueHeap.next(KeyValueHeap.java:155) at org.apache.hadoop.hbase.regionserver.HRegion$RegionScannerImpl.populateResult(HRegion.java:7399) at org.apache.hadoop.hbase.regionserver.HRegion$RegionScannerImpl.nextInternal(HRegion.java:7567) at org.apache.hadoop.hbase.regionserver.HRegion$RegionScannerImpl.nextRaw(HRegion.java:7331) at org.apache.hadoop.hbase.regionserver.HRegion$RegionScannerImpl.nextRaw(HRegion.java:7317) at org.apache.phoenix.coprocessor.PagedRegionScanner.next(PagedRegionScanner.java:57) at org.apache.phoenix.coprocessor.PagedRegionScanner.nextRaw(PagedRegionScanner.java:96) at org.apache.phoenix.iterate.RegionScannerFactory$1.nextRaw(RegionScannerFactory.java:211) at org.apache.phoenix.coprocessor.DelegateRegionScanner.nextRaw(DelegateRegionScanner.java:77) at org.apache.phoenix.coprocessor.DelegateRegionScanner.nextRaw(DelegateRegionScanner.java:77) at org.apache.phoenix.coprocessor.BaseScannerRegionObserver$RegionScannerHolder.nextRaw(BaseScannerRegionObserver.java:308) at org.apache.hadoop.hbase.regionserver.RSRpcServices.scan(RSRpcServices.java:3373) - locked <0x00007fc85f0278e0> (a org.apache.phoenix.coprocessor.BaseScannerRegionObserver$RegionScannerHolder) at org.apache.hadoop.hbase.regionserver.RSRpcServices.scan(RSRpcServices.java:3637) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45819) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:392) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:140) at org.apache.hadoop.hbase.ipc.RpcExecutor$Handler.run(RpcExecutor.java:359) at org.apache.hadoop.hbase.ipc.RpcExecutor$Handler.run(RpcExecutor.java:339) "RpcServer.default.FPBQ.Fifo.handler=84,queue=4,port=16020" #171 daemon prio=5 os_prio=0 tid=0x00007fc11d89c800 nid=0x7420 waiting on condition [0x00007fc1126b9000] java.lang.Thread.State: WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007fc1c9fe8f38> (a java.util.concurrent.Semaphore$NonfairSync) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer.parkAndCheckInterrupt(AbstractQueuedSynchronizer.java:836) at java.util.concurrent.locks.AbstractQueuedSynchronizer.doAcquireSharedInterruptibly(AbstractQueuedSynchronizer.java:997) at java.util.concurrent.locks.AbstractQueuedSynchronizer.acquireSharedInterruptibly(AbstractQueuedSynchronizer.java:1304) at java.util.concurrent.Semaphore.acquire(Semaphore.java:312) at org.apache.hadoop.hbase.ipc.FastPathBalancedQueueRpcExecutor$FastPathHandler.getCallRunner(FastPathBalancedQueueRpcExecutor.java:109) at org.apache.hadoop.hbase.ipc.RpcExecutor$Handler.run(RpcExecutor.java:339) "RpcServer.default.FPBQ.Fifo.handler=83,queue=3,port=16020" #170 daemon prio=5 os_prio=0 tid=0x00007fc11d89a800 nid=0x741f waiting for monitor entry [0x00007fc1127b9000] java.lang.Thread.State: BLOCKED (on object monitor) at org.apache.hadoop.fs.FSInputStream.read(FSInputStream.java:73) - waiting to lock <0x00007fc4c2769660> (a org.apache.hadoop.fs.s3a.S3AInputStream) at org.apache.hadoop.fs.FSDataInputStream.read(FSDataInputStream.java:97) at org.apache.hadoop.hbase.io.FileLink$FileLinkInputStream.read(FileLink.java:171) at org.apache.hadoop.fs.FSDataInputStream.read(FSDataInputStream.java:97) at org.apache.hadoop.hbase.io.util.BlockIOUtils.preadWithExtra(BlockIOUtils.java:235) at org.apache.hadoop.hbase.io.hfile.HFileBlock$FSReaderImpl.readAtOffset(HFileBlock.java:1564) at org.apache.hadoop.hbase.io.hfile.HFileBlock$FSReaderImpl.readBlockDataInternal(HFileBlock.java:1788) at org.apache.hadoop.hbase.io.hfile.HFileBlock$FSReaderImpl.readBlockData(HFileBlock.java:1597) at org.apache.hadoop.hbase.io.hfile.HFileReaderImpl.readBlock(HFileReaderImpl.java:1347) at org.apache.hadoop.hbase.io.hfile.HFileReaderImpl.readBlock(HFileReaderImpl.java:1268) at org.apache.hadoop.hbase.io.hfile.HFileReaderImpl$HFileScannerImpl.readNextDataBlock(HFileReaderImpl.java:762) at org.apache.hadoop.hbase.io.hfile.HFileReaderImpl$EncodedScanner.next(HFileReaderImpl.java:1540) at org.apache.hadoop.hbase.regionserver.StoreFileScanner.reseekAtOrAfter(StoreFileScanner.java:351) at org.apache.hadoop.hbase.regionserver.StoreFileScanner.reseek(StoreFileScanner.java:244) at org.apache.hadoop.hbase.regionserver.NonLazyKeyValueScanner.doRealSeek(NonLazyKeyValueScanner.java:55) at org.apache.hadoop.hbase.regionserver.KeyValueHeap.generalizedSeek(KeyValueHeap.java:322) at org.apache.hadoop.hbase.regionserver.KeyValueHeap.reseek(KeyValueHeap.java:265) at org.apache.hadoop.hbase.regionserver.StoreScanner.reseek(StoreScanner.java:1095) at org.apache.hadoop.hbase.regionserver.StoreScanner.seekAsDirection(StoreScanner.java:1084) at org.apache.hadoop.hbase.regionserver.StoreScanner.seekOrSkipToNextColumn(StoreScanner.java:821) at org.apache.hadoop.hbase.regionserver.StoreScanner.next(StoreScanner.java:684) at org.apache.hadoop.hbase.regionserver.KeyValueHeap.next(KeyValueHeap.java:155) at org.apache.hadoop.hbase.regionserver.HRegion$RegionScannerImpl.populateResult(HRegion.java:7399) at org.apache.hadoop.hbase.regionserver.HRegion$RegionScannerImpl.nextInternal(HRegion.java:7567) at org.apache.hadoop.hbase.regionserver.HRegion$RegionScannerImpl.nextRaw(HRegion.java:7331) at org.apache.hadoop.hbase.regionserver.HRegion$RegionScannerImpl.nextRaw(HRegion.java:7317) at org.apache.phoenix.coprocessor.PagedRegionScanner.next(PagedRegionScanner.java:57) at org.apache.phoenix.coprocessor.PagedRegionScanner.nextRaw(PagedRegionScanner.java:96) at org.apache.phoenix.iterate.RegionScannerFactory$1.nextRaw(RegionScannerFactory.java:211) at org.apache.phoenix.coprocessor.DelegateRegionScanner.nextRaw(DelegateRegionScanner.java:77) at org.apache.phoenix.coprocessor.DelegateRegionScanner.nextRaw(DelegateRegionScanner.java:77) at org.apache.phoenix.coprocessor.BaseScannerRegionObserver$RegionScannerHolder.nextRaw(BaseScannerRegionObserver.java:308) at org.apache.hadoop.hbase.regionserver.RSRpcServices.scan(RSRpcServices.java:3373) - locked <0x00007fc6b139ba68> (a org.apache.phoenix.coprocessor.BaseScannerRegionObserver$RegionScannerHolder) at org.apache.hadoop.hbase.regionserver.RSRpcServices.scan(RSRpcServices.java:3637) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45819) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:392) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:140) at org.apache.hadoop.hbase.ipc.RpcExecutor$Handler.run(RpcExecutor.java:359) at org.apache.hadoop.hbase.ipc.RpcExecutor$Handler.run(RpcExecutor.java:339) "RpcServer.default.FPBQ.Fifo.handler=82,queue=2,port=16020" #169 daemon prio=5 os_prio=0 tid=0x00007fc11d898800 nid=0x741e waiting for monitor entry [0x00007fc1128ba000] java.lang.Thread.State: BLOCKED (on object monitor) at org.apache.hadoop.fs.FSInputStream.read(FSInputStream.java:73) - waiting to lock <0x00007fc4c2769660> (a org.apache.hadoop.fs.s3a.S3AInputStream) at org.apache.hadoop.fs.FSDataInputStream.read(FSDataInputStream.java:97) at org.apache.hadoop.hbase.io.FileLink$FileLinkInputStream.read(FileLink.java:171) at org.apache.hadoop.fs.FSDataInputStream.read(FSDataInputStream.java:97) at org.apache.hadoop.hbase.io.util.BlockIOUtils.preadWithExtra(BlockIOUtils.java:235) at org.apache.hadoop.hbase.io.hfile.HFileBlock$FSReaderImpl.readAtOffset(HFileBlock.java:1564) at org.apache.hadoop.hbase.io.hfile.HFileBlock$FSReaderImpl.readBlockDataInternal(HFileBlock.java:1788) at org.apache.hadoop.hbase.io.hfile.HFileBlock$FSReaderImpl.readBlockData(HFileBlock.java:1597) at org.apache.hadoop.hbase.io.hfile.HFileReaderImpl.readBlock(HFileReaderImpl.java:1347) at org.apache.hadoop.hbase.io.hfile.HFileReaderImpl.readBlock(HFileReaderImpl.java:1268) at org.apache.hadoop.hbase.io.hfile.HFileReaderImpl$HFileScannerImpl.readNextDataBlock(HFileReaderImpl.java:762) at org.apache.hadoop.hbase.io.hfile.HFileReaderImpl$EncodedScanner.next(HFileReaderImpl.java:1540) at org.apache.hadoop.hbase.regionserver.StoreFileScanner.reseekAtOrAfter(StoreFileScanner.java:351) at org.apache.hadoop.hbase.regionserver.StoreFileScanner.reseek(StoreFileScanner.java:244) at org.apache.hadoop.hbase.regionserver.NonLazyKeyValueScanner.doRealSeek(NonLazyKeyValueScanner.java:55) at org.apache.hadoop.hbase.regionserver.KeyValueHeap.generalizedSeek(KeyValueHeap.java:322) at org.apache.hadoop.hbase.regionserver.KeyValueHeap.reseek(KeyValueHeap.java:265) at org.apache.hadoop.hbase.regionserver.StoreScanner.reseek(StoreScanner.java:1095) at org.apache.hadoop.hbase.regionserver.StoreScanner.seekAsDirection(StoreScanner.java:1084) at org.apache.hadoop.hbase.regionserver.StoreScanner.seekOrSkipToNextColumn(StoreScanner.java:821) at org.apache.hadoop.hbase.regionserver.StoreScanner.next(StoreScanner.java:684) at org.apache.hadoop.hbase.regionserver.KeyValueHeap.next(KeyValueHeap.java:155) at org.apache.hadoop.hbase.regionserver.HRegion$RegionScannerImpl.populateResult(HRegion.java:7399) at org.apache.hadoop.hbase.regionserver.HRegion$RegionScannerImpl.nextInternal(HRegion.java:7567) at org.apache.hadoop.hbase.regionserver.HRegion$RegionScannerImpl.nextRaw(HRegion.java:7331) at org.apache.hadoop.hbase.regionserver.HRegion$RegionScannerImpl.nextRaw(HRegion.java:7317) at org.apache.phoenix.coprocessor.PagedRegionScanner.next(PagedRegionScanner.java:57) at org.apache.phoenix.coprocessor.PagedRegionScanner.nextRaw(PagedRegionScanner.java:96) at org.apache.phoenix.iterate.RegionScannerFactory$1.nextRaw(RegionScannerFactory.java:211) at org.apache.phoenix.coprocessor.DelegateRegionScanner.nextRaw(DelegateRegionScanner.java:77) at org.apache.phoenix.coprocessor.DelegateRegionScanner.nextRaw(DelegateRegionScanner.java:77) at org.apache.phoenix.coprocessor.BaseScannerRegionObserver$RegionScannerHolder.nextRaw(BaseScannerRegionObserver.java:308) at org.apache.hadoop.hbase.regionserver.RSRpcServices.scan(RSRpcServices.java:3373) - locked <0x00007fc85f007788> (a org.apache.phoenix.coprocessor.BaseScannerRegionObserver$RegionScannerHolder) at org.apache.hadoop.hbase.regionserver.RSRpcServices.scan(RSRpcServices.java:3637) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45819) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:392) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:140) at org.apache.hadoop.hbase.ipc.RpcExecutor$Handler.run(RpcExecutor.java:359) at org.apache.hadoop.hbase.ipc.RpcExecutor$Handler.run(RpcExecutor.java:339) "RpcServer.default.FPBQ.Fifo.handler=81,queue=1,port=16020" #168 daemon prio=5 os_prio=0 tid=0x00007fc11d896800 nid=0x741d waiting for monitor entry [0x00007fc1129bb000] java.lang.Thread.State: BLOCKED (on object monitor) at org.apache.hadoop.fs.FSInputStream.read(FSInputStream.java:73) - waiting to lock <0x00007fc4c2769660> (a org.apache.hadoop.fs.s3a.S3AInputStream) at org.apache.hadoop.fs.FSDataInputStream.read(FSDataInputStream.java:97) at org.apache.hadoop.hbase.io.FileLink$FileLinkInputStream.read(FileLink.java:171) at org.apache.hadoop.fs.FSDataInputStream.read(FSDataInputStream.java:97) at org.apache.hadoop.hbase.io.util.BlockIOUtils.preadWithExtra(BlockIOUtils.java:235) at org.apache.hadoop.hbase.io.hfile.HFileBlock$FSReaderImpl.readAtOffset(HFileBlock.java:1564) at org.apache.hadoop.hbase.io.hfile.HFileBlock$FSReaderImpl.readBlockDataInternal(HFileBlock.java:1788) at org.apache.hadoop.hbase.io.hfile.HFileBlock$FSReaderImpl.readBlockData(HFileBlock.java:1597) at org.apache.hadoop.hbase.io.hfile.HFileReaderImpl.readBlock(HFileReaderImpl.java:1347) at org.apache.hadoop.hbase.io.hfile.HFileReaderImpl.readBlock(HFileReaderImpl.java:1268) at org.apache.hadoop.hbase.io.hfile.HFileReaderImpl$HFileScannerImpl.readNextDataBlock(HFileReaderImpl.java:762) at org.apache.hadoop.hbase.io.hfile.HFileReaderImpl$EncodedScanner.next(HFileReaderImpl.java:1540) at org.apache.hadoop.hbase.regionserver.StoreFileScanner.reseekAtOrAfter(StoreFileScanner.java:351) at org.apache.hadoop.hbase.regionserver.StoreFileScanner.reseek(StoreFileScanner.java:244) at org.apache.hadoop.hbase.regionserver.NonLazyKeyValueScanner.doRealSeek(NonLazyKeyValueScanner.java:55) at org.apache.hadoop.hbase.regionserver.KeyValueHeap.generalizedSeek(KeyValueHeap.java:322) at org.apache.hadoop.hbase.regionserver.KeyValueHeap.reseek(KeyValueHeap.java:265) at org.apache.hadoop.hbase.regionserver.StoreScanner.reseek(StoreScanner.java:1095) at org.apache.hadoop.hbase.regionserver.StoreScanner.seekAsDirection(StoreScanner.java:1084) at org.apache.hadoop.hbase.regionserver.StoreScanner.seekOrSkipToNextColumn(StoreScanner.java:821) at org.apache.hadoop.hbase.regionserver.StoreScanner.next(StoreScanner.java:684) at org.apache.hadoop.hbase.regionserver.KeyValueHeap.next(KeyValueHeap.java:155) at org.apache.hadoop.hbase.regionserver.HRegion$RegionScannerImpl.populateResult(HRegion.java:7399) at org.apache.hadoop.hbase.regionserver.HRegion$RegionScannerImpl.nextInternal(HRegion.java:7567) at org.apache.hadoop.hbase.regionserver.HRegion$RegionScannerImpl.nextRaw(HRegion.java:7331) at org.apache.hadoop.hbase.regionserver.HRegion$RegionScannerImpl.nextRaw(HRegion.java:7317) at org.apache.phoenix.coprocessor.PagedRegionScanner.next(PagedRegionScanner.java:57) at org.apache.phoenix.coprocessor.PagedRegionScanner.nextRaw(PagedRegionScanner.java:96) at org.apache.phoenix.iterate.RegionScannerFactory$1.nextRaw(RegionScannerFactory.java:211) at org.apache.phoenix.coprocessor.DelegateRegionScanner.nextRaw(DelegateRegionScanner.java:77) at org.apache.phoenix.coprocessor.DelegateRegionScanner.nextRaw(DelegateRegionScanner.java:77) at org.apache.phoenix.coprocessor.BaseScannerRegionObserver$RegionScannerHolder.nextRaw(BaseScannerRegionObserver.java:308) at org.apache.hadoop.hbase.regionserver.RSRpcServices.scan(RSRpcServices.java:3373) - locked <0x00007fc85f027e58> (a org.apache.phoenix.coprocessor.BaseScannerRegionObserver$RegionScannerHolder) at org.apache.hadoop.hbase.regionserver.RSRpcServices.scan(RSRpcServices.java:3637) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45819) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:392) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:140) at org.apache.hadoop.hbase.ipc.RpcExecutor$Handler.run(RpcExecutor.java:359) at org.apache.hadoop.hbase.ipc.RpcExecutor$Handler.run(RpcExecutor.java:339) "RpcServer.default.FPBQ.Fifo.handler=80,queue=0,port=16020" #167 daemon prio=5 os_prio=0 tid=0x00007fc11d894000 nid=0x741c waiting for monitor entry [0x00007fc112abc000] java.lang.Thread.State: BLOCKED (on object monitor) at org.apache.hadoop.fs.FSInputStream.read(FSInputStream.java:73) - waiting to lock <0x00007fc4c2769660> (a org.apache.hadoop.fs.s3a.S3AInputStream) at org.apache.hadoop.fs.FSDataInputStream.read(FSDataInputStream.java:97) at org.apache.hadoop.hbase.io.FileLink$FileLinkInputStream.read(FileLink.java:171) at org.apache.hadoop.fs.FSDataInputStream.read(FSDataInputStream.java:97) at org.apache.hadoop.hbase.io.util.BlockIOUtils.preadWithExtra(BlockIOUtils.java:235) at org.apache.hadoop.hbase.io.hfile.HFileBlock$FSReaderImpl.readAtOffset(HFileBlock.java:1564) at org.apache.hadoop.hbase.io.hfile.HFileBlock$FSReaderImpl.readBlockDataInternal(HFileBlock.java:1788) at org.apache.hadoop.hbase.io.hfile.HFileBlock$FSReaderImpl.readBlockData(HFileBlock.java:1597) at org.apache.hadoop.hbase.io.hfile.HFileReaderImpl.readBlock(HFileReaderImpl.java:1347) at org.apache.hadoop.hbase.io.hfile.HFileReaderImpl.readBlock(HFileReaderImpl.java:1268) at org.apache.hadoop.hbase.io.hfile.HFileReaderImpl$HFileScannerImpl.readNextDataBlock(HFileReaderImpl.java:762) at org.apache.hadoop.hbase.io.hfile.HFileReaderImpl$EncodedScanner.next(HFileReaderImpl.java:1540) at org.apache.hadoop.hbase.regionserver.StoreFileScanner.reseekAtOrAfter(StoreFileScanner.java:351) at org.apache.hadoop.hbase.regionserver.StoreFileScanner.reseek(StoreFileScanner.java:244) at org.apache.hadoop.hbase.regionserver.NonLazyKeyValueScanner.doRealSeek(NonLazyKeyValueScanner.java:55) at org.apache.hadoop.hbase.regionserver.KeyValueHeap.generalizedSeek(KeyValueHeap.java:322) at org.apache.hadoop.hbase.regionserver.KeyValueHeap.reseek(KeyValueHeap.java:265) at org.apache.hadoop.hbase.regionserver.StoreScanner.reseek(StoreScanner.java:1095) at org.apache.hadoop.hbase.regionserver.StoreScanner.seekAsDirection(StoreScanner.java:1084) at org.apache.hadoop.hbase.regionserver.StoreScanner.seekOrSkipToNextColumn(StoreScanner.java:821) at org.apache.hadoop.hbase.regionserver.StoreScanner.next(StoreScanner.java:684) at org.apache.hadoop.hbase.regionserver.KeyValueHeap.next(KeyValueHeap.java:155) at org.apache.hadoop.hbase.regionserver.HRegion$RegionScannerImpl.populateResult(HRegion.java:7399) at org.apache.hadoop.hbase.regionserver.HRegion$RegionScannerImpl.nextInternal(HRegion.java:7567) at org.apache.hadoop.hbase.regionserver.HRegion$RegionScannerImpl.nextRaw(HRegion.java:7331) at org.apache.hadoop.hbase.regionserver.HRegion$RegionScannerImpl.nextRaw(HRegion.java:7317) at org.apache.phoenix.coprocessor.PagedRegionScanner.next(PagedRegionScanner.java:57) at org.apache.phoenix.coprocessor.PagedRegionScanner.nextRaw(PagedRegionScanner.java:96) at org.apache.phoenix.iterate.RegionScannerFactory$1.nextRaw(RegionScannerFactory.java:211) at org.apache.phoenix.coprocessor.DelegateRegionScanner.nextRaw(DelegateRegionScanner.java:77) at org.apache.phoenix.coprocessor.DelegateRegionScanner.nextRaw(DelegateRegionScanner.java:77) at org.apache.phoenix.coprocessor.BaseScannerRegionObserver$RegionScannerHolder.nextRaw(BaseScannerRegionObserver.java:308) at org.apache.hadoop.hbase.regionserver.RSRpcServices.scan(RSRpcServices.java:3373) - locked <0x00007fc6b139bee8> (a org.apache.phoenix.coprocessor.BaseScannerRegionObserver$RegionScannerHolder) at org.apache.hadoop.hbase.regionserver.RSRpcServices.scan(RSRpcServices.java:3637) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45819) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:392) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:140) at org.apache.hadoop.hbase.ipc.RpcExecutor$Handler.run(RpcExecutor.java:359) at org.apache.hadoop.hbase.ipc.RpcExecutor$Handler.run(RpcExecutor.java:339) "RpcServer.default.FPBQ.Fifo.handler=79,queue=9,port=16020" #166 daemon prio=5 os_prio=0 tid=0x00007fc11d892000 nid=0x741b waiting on condition [0x00007fc112bbe000] java.lang.Thread.State: WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007fc1c9fe9148> (a java.util.concurrent.Semaphore$NonfairSync) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer.parkAndCheckInterrupt(AbstractQueuedSynchronizer.java:836) at java.util.concurrent.locks.AbstractQueuedSynchronizer.doAcquireSharedInterruptibly(AbstractQueuedSynchronizer.java:997) at java.util.concurrent.locks.AbstractQueuedSynchronizer.acquireSharedInterruptibly(AbstractQueuedSynchronizer.java:1304) at java.util.concurrent.Semaphore.acquire(Semaphore.java:312) at org.apache.hadoop.hbase.ipc.FastPathBalancedQueueRpcExecutor$FastPathHandler.getCallRunner(FastPathBalancedQueueRpcExecutor.java:109) at org.apache.hadoop.hbase.ipc.RpcExecutor$Handler.run(RpcExecutor.java:339) "RpcServer.default.FPBQ.Fifo.handler=78,queue=8,port=16020" #165 daemon prio=5 os_prio=0 tid=0x00007fc11d890000 nid=0x741a waiting on condition [0x00007fc112cbf000] java.lang.Thread.State: WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007fc1c9fe3868> (a java.util.concurrent.Semaphore$NonfairSync) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer.parkAndCheckInterrupt(AbstractQueuedSynchronizer.java:836) at java.util.concurrent.locks.AbstractQueuedSynchronizer.doAcquireSharedInterruptibly(AbstractQueuedSynchronizer.java:997) at java.util.concurrent.locks.AbstractQueuedSynchronizer.acquireSharedInterruptibly(AbstractQueuedSynchronizer.java:1304) at java.util.concurrent.Semaphore.acquire(Semaphore.java:312) at org.apache.hadoop.hbase.ipc.FastPathBalancedQueueRpcExecutor$FastPathHandler.getCallRunner(FastPathBalancedQueueRpcExecutor.java:109) at org.apache.hadoop.hbase.ipc.RpcExecutor$Handler.run(RpcExecutor.java:339) "RpcServer.default.FPBQ.Fifo.handler=77,queue=7,port=16020" #164 daemon prio=5 os_prio=0 tid=0x00007fc11d88e000 nid=0x7419 waiting for monitor entry [0x00007fc112dbf000] java.lang.Thread.State: BLOCKED (on object monitor) at org.apache.hadoop.fs.FSInputStream.read(FSInputStream.java:73) - waiting to lock <0x00007fc4c2769660> (a org.apache.hadoop.fs.s3a.S3AInputStream) at org.apache.hadoop.fs.FSDataInputStream.read(FSDataInputStream.java:97) at org.apache.hadoop.hbase.io.FileLink$FileLinkInputStream.read(FileLink.java:171) at org.apache.hadoop.fs.FSDataInputStream.read(FSDataInputStream.java:97) at org.apache.hadoop.hbase.io.util.BlockIOUtils.preadWithExtra(BlockIOUtils.java:235) at org.apache.hadoop.hbase.io.hfile.HFileBlock$FSReaderImpl.readAtOffset(HFileBlock.java:1564) at org.apache.hadoop.hbase.io.hfile.HFileBlock$FSReaderImpl.readBlockDataInternal(HFileBlock.java:1788) at org.apache.hadoop.hbase.io.hfile.HFileBlock$FSReaderImpl.readBlockData(HFileBlock.java:1597) at org.apache.hadoop.hbase.io.hfile.HFileReaderImpl.readBlock(HFileReaderImpl.java:1347) at org.apache.hadoop.hbase.io.hfile.HFileReaderImpl.readBlock(HFileReaderImpl.java:1268) at org.apache.hadoop.hbase.io.hfile.HFileReaderImpl$HFileScannerImpl.readNextDataBlock(HFileReaderImpl.java:762) at org.apache.hadoop.hbase.io.hfile.HFileReaderImpl$EncodedScanner.next(HFileReaderImpl.java:1540) at org.apache.hadoop.hbase.regionserver.StoreFileScanner.reseekAtOrAfter(StoreFileScanner.java:351) at org.apache.hadoop.hbase.regionserver.StoreFileScanner.reseek(StoreFileScanner.java:244) at org.apache.hadoop.hbase.regionserver.NonLazyKeyValueScanner.doRealSeek(NonLazyKeyValueScanner.java:55) at org.apache.hadoop.hbase.regionserver.KeyValueHeap.generalizedSeek(KeyValueHeap.java:322) at org.apache.hadoop.hbase.regionserver.KeyValueHeap.reseek(KeyValueHeap.java:265) at org.apache.hadoop.hbase.regionserver.StoreScanner.reseek(StoreScanner.java:1095) at org.apache.hadoop.hbase.regionserver.StoreScanner.seekAsDirection(StoreScanner.java:1084) at org.apache.hadoop.hbase.regionserver.StoreScanner.seekOrSkipToNextColumn(StoreScanner.java:821) at org.apache.hadoop.hbase.regionserver.StoreScanner.next(StoreScanner.java:684) at org.apache.hadoop.hbase.regionserver.KeyValueHeap.next(KeyValueHeap.java:155) at org.apache.hadoop.hbase.regionserver.HRegion$RegionScannerImpl.populateResult(HRegion.java:7399) at org.apache.hadoop.hbase.regionserver.HRegion$RegionScannerImpl.nextInternal(HRegion.java:7567) at org.apache.hadoop.hbase.regionserver.HRegion$RegionScannerImpl.nextRaw(HRegion.java:7331) at org.apache.hadoop.hbase.regionserver.HRegion$RegionScannerImpl.nextRaw(HRegion.java:7317) at org.apache.phoenix.coprocessor.PagedRegionScanner.next(PagedRegionScanner.java:57) at org.apache.phoenix.coprocessor.PagedRegionScanner.nextRaw(PagedRegionScanner.java:96) at org.apache.phoenix.iterate.RegionScannerFactory$1.nextRaw(RegionScannerFactory.java:211) at org.apache.phoenix.coprocessor.DelegateRegionScanner.nextRaw(DelegateRegionScanner.java:77) at org.apache.phoenix.coprocessor.DelegateRegionScanner.nextRaw(DelegateRegionScanner.java:77) at org.apache.phoenix.coprocessor.BaseScannerRegionObserver$RegionScannerHolder.nextRaw(BaseScannerRegionObserver.java:308) at org.apache.hadoop.hbase.regionserver.RSRpcServices.scan(RSRpcServices.java:3373) - locked <0x00007fc85f02f438> (a org.apache.phoenix.coprocessor.BaseScannerRegionObserver$RegionScannerHolder) at org.apache.hadoop.hbase.regionserver.RSRpcServices.scan(RSRpcServices.java:3637) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45819) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:392) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:140) at org.apache.hadoop.hbase.ipc.RpcExecutor$Handler.run(RpcExecutor.java:359) at org.apache.hadoop.hbase.ipc.RpcExecutor$Handler.run(RpcExecutor.java:339) "RpcServer.default.FPBQ.Fifo.handler=76,queue=6,port=16020" #163 daemon prio=5 os_prio=0 tid=0x00007fc11d88b800 nid=0x7418 waiting for monitor entry [0x00007fc112ec0000] java.lang.Thread.State: BLOCKED (on object monitor) at org.apache.hadoop.fs.FSInputStream.read(FSInputStream.java:73) - waiting to lock <0x00007fc4c2769660> (a org.apache.hadoop.fs.s3a.S3AInputStream) at org.apache.hadoop.fs.FSDataInputStream.read(FSDataInputStream.java:97) at org.apache.hadoop.hbase.io.FileLink$FileLinkInputStream.read(FileLink.java:171) at org.apache.hadoop.fs.FSDataInputStream.read(FSDataInputStream.java:97) at org.apache.hadoop.hbase.io.util.BlockIOUtils.preadWithExtra(BlockIOUtils.java:235) at org.apache.hadoop.hbase.io.hfile.HFileBlock$FSReaderImpl.readAtOffset(HFileBlock.java:1564) at org.apache.hadoop.hbase.io.hfile.HFileBlock$FSReaderImpl.readBlockDataInternal(HFileBlock.java:1788) at org.apache.hadoop.hbase.io.hfile.HFileBlock$FSReaderImpl.readBlockData(HFileBlock.java:1597) at org.apache.hadoop.hbase.io.hfile.HFileReaderImpl.readBlock(HFileReaderImpl.java:1347) at org.apache.hadoop.hbase.io.hfile.HFileReaderImpl.readBlock(HFileReaderImpl.java:1268) at org.apache.hadoop.hbase.io.hfile.HFileReaderImpl$HFileScannerImpl.readNextDataBlock(HFileReaderImpl.java:762) at org.apache.hadoop.hbase.io.hfile.HFileReaderImpl$EncodedScanner.next(HFileReaderImpl.java:1540) at org.apache.hadoop.hbase.regionserver.StoreFileScanner.reseekAtOrAfter(StoreFileScanner.java:351) at org.apache.hadoop.hbase.regionserver.StoreFileScanner.reseek(StoreFileScanner.java:244) at org.apache.hadoop.hbase.regionserver.NonLazyKeyValueScanner.doRealSeek(NonLazyKeyValueScanner.java:55) at org.apache.hadoop.hbase.regionserver.KeyValueHeap.generalizedSeek(KeyValueHeap.java:322) at org.apache.hadoop.hbase.regionserver.KeyValueHeap.reseek(KeyValueHeap.java:265) at org.apache.hadoop.hbase.regionserver.StoreScanner.reseek(StoreScanner.java:1095) at org.apache.hadoop.hbase.regionserver.StoreScanner.seekAsDirection(StoreScanner.java:1084) at org.apache.hadoop.hbase.regionserver.StoreScanner.seekOrSkipToNextColumn(StoreScanner.java:821) at org.apache.hadoop.hbase.regionserver.StoreScanner.next(StoreScanner.java:684) at org.apache.hadoop.hbase.regionserver.KeyValueHeap.next(KeyValueHeap.java:155) at org.apache.hadoop.hbase.regionserver.HRegion$RegionScannerImpl.populateResult(HRegion.java:7399) at org.apache.hadoop.hbase.regionserver.HRegion$RegionScannerImpl.nextInternal(HRegion.java:7567) at org.apache.hadoop.hbase.regionserver.HRegion$RegionScannerImpl.nextRaw(HRegion.java:7331) at org.apache.hadoop.hbase.regionserver.HRegion$RegionScannerImpl.nextRaw(HRegion.java:7317) at org.apache.phoenix.coprocessor.PagedRegionScanner.next(PagedRegionScanner.java:57) at org.apache.phoenix.coprocessor.PagedRegionScanner.nextRaw(PagedRegionScanner.java:96) at org.apache.phoenix.iterate.RegionScannerFactory$1.nextRaw(RegionScannerFactory.java:211) at org.apache.phoenix.coprocessor.DelegateRegionScanner.nextRaw(DelegateRegionScanner.java:77) at org.apache.phoenix.coprocessor.DelegateRegionScanner.nextRaw(DelegateRegionScanner.java:77) at org.apache.phoenix.coprocessor.BaseScannerRegionObserver$RegionScannerHolder.nextRaw(BaseScannerRegionObserver.java:308) at org.apache.hadoop.hbase.regionserver.RSRpcServices.scan(RSRpcServices.java:3373) - locked <0x00007fc85f007d70> (a org.apache.phoenix.coprocessor.BaseScannerRegionObserver$RegionScannerHolder) at org.apache.hadoop.hbase.regionserver.RSRpcServices.scan(RSRpcServices.java:3637) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45819) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:392) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:140) at org.apache.hadoop.hbase.ipc.RpcExecutor$Handler.run(RpcExecutor.java:359) at org.apache.hadoop.hbase.ipc.RpcExecutor$Handler.run(RpcExecutor.java:339) "RpcServer.default.FPBQ.Fifo.handler=75,queue=5,port=16020" #162 daemon prio=5 os_prio=0 tid=0x00007fc11d889800 nid=0x7417 waiting for monitor entry [0x00007fc112fc1000] java.lang.Thread.State: BLOCKED (on object monitor) at org.apache.hadoop.fs.FSInputStream.read(FSInputStream.java:73) - waiting to lock <0x00007fc4c2769660> (a org.apache.hadoop.fs.s3a.S3AInputStream) at org.apache.hadoop.fs.FSDataInputStream.read(FSDataInputStream.java:97) at org.apache.hadoop.hbase.io.FileLink$FileLinkInputStream.read(FileLink.java:171) at org.apache.hadoop.fs.FSDataInputStream.read(FSDataInputStream.java:97) at org.apache.hadoop.hbase.io.util.BlockIOUtils.preadWithExtra(BlockIOUtils.java:235) at org.apache.hadoop.hbase.io.hfile.HFileBlock$FSReaderImpl.readAtOffset(HFileBlock.java:1564) at org.apache.hadoop.hbase.io.hfile.HFileBlock$FSReaderImpl.readBlockDataInternal(HFileBlock.java:1788) at org.apache.hadoop.hbase.io.hfile.HFileBlock$FSReaderImpl.readBlockData(HFileBlock.java:1597) at org.apache.hadoop.hbase.io.hfile.HFileReaderImpl.readBlock(HFileReaderImpl.java:1347) at org.apache.hadoop.hbase.io.hfile.HFileReaderImpl.readBlock(HFileReaderImpl.java:1268) at org.apache.hadoop.hbase.io.hfile.HFileReaderImpl$HFileScannerImpl.readNextDataBlock(HFileReaderImpl.java:762) at org.apache.hadoop.hbase.io.hfile.HFileReaderImpl$EncodedScanner.next(HFileReaderImpl.java:1540) at org.apache.hadoop.hbase.regionserver.StoreFileScanner.reseekAtOrAfter(StoreFileScanner.java:351) at org.apache.hadoop.hbase.regionserver.StoreFileScanner.reseek(StoreFileScanner.java:244) at org.apache.hadoop.hbase.regionserver.NonLazyKeyValueScanner.doRealSeek(NonLazyKeyValueScanner.java:55) at org.apache.hadoop.hbase.regionserver.KeyValueHeap.generalizedSeek(KeyValueHeap.java:322) at org.apache.hadoop.hbase.regionserver.KeyValueHeap.reseek(KeyValueHeap.java:265) at org.apache.hadoop.hbase.regionserver.StoreScanner.reseek(StoreScanner.java:1095) at org.apache.hadoop.hbase.regionserver.StoreScanner.seekAsDirection(StoreScanner.java:1084) at org.apache.hadoop.hbase.regionserver.StoreScanner.seekOrSkipToNextColumn(StoreScanner.java:821) at org.apache.hadoop.hbase.regionserver.StoreScanner.next(StoreScanner.java:684) at org.apache.hadoop.hbase.regionserver.KeyValueHeap.next(KeyValueHeap.java:155) at org.apache.hadoop.hbase.regionserver.HRegion$RegionScannerImpl.populateResult(HRegion.java:7399) at org.apache.hadoop.hbase.regionserver.HRegion$RegionScannerImpl.nextInternal(HRegion.java:7567) at org.apache.hadoop.hbase.regionserver.HRegion$RegionScannerImpl.nextRaw(HRegion.java:7331) at org.apache.hadoop.hbase.regionserver.HRegion$RegionScannerImpl.nextRaw(HRegion.java:7317) at org.apache.phoenix.coprocessor.PagedRegionScanner.next(PagedRegionScanner.java:57) at org.apache.phoenix.coprocessor.PagedRegionScanner.nextRaw(PagedRegionScanner.java:96) at org.apache.phoenix.iterate.RegionScannerFactory$1.nextRaw(RegionScannerFactory.java:211) at org.apache.phoenix.coprocessor.DelegateRegionScanner.nextRaw(DelegateRegionScanner.java:77) at org.apache.phoenix.coprocessor.DelegateRegionScanner.nextRaw(DelegateRegionScanner.java:77) at org.apache.phoenix.coprocessor.BaseScannerRegionObserver$RegionScannerHolder.nextRaw(BaseScannerRegionObserver.java:308) at org.apache.hadoop.hbase.regionserver.RSRpcServices.scan(RSRpcServices.java:3373) - locked <0x00007fc85f051420> (a org.apache.phoenix.coprocessor.BaseScannerRegionObserver$RegionScannerHolder) at org.apache.hadoop.hbase.regionserver.RSRpcServices.scan(RSRpcServices.java:3637) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45819) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:392) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:140) at org.apache.hadoop.hbase.ipc.RpcExecutor$Handler.run(RpcExecutor.java:359) at org.apache.hadoop.hbase.ipc.RpcExecutor$Handler.run(RpcExecutor.java:339) "RpcServer.default.FPBQ.Fifo.handler=74,queue=4,port=16020" #161 daemon prio=5 os_prio=0 tid=0x00007fc11d887800 nid=0x7416 waiting on condition [0x00007fc1130c3000] java.lang.Thread.State: WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007fc1c9fe9358> (a java.util.concurrent.Semaphore$NonfairSync) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer.parkAndCheckInterrupt(AbstractQueuedSynchronizer.java:836) at java.util.concurrent.locks.AbstractQueuedSynchronizer.doAcquireSharedInterruptibly(AbstractQueuedSynchronizer.java:997) at java.util.concurrent.locks.AbstractQueuedSynchronizer.acquireSharedInterruptibly(AbstractQueuedSynchronizer.java:1304) at java.util.concurrent.Semaphore.acquire(Semaphore.java:312) at org.apache.hadoop.hbase.ipc.FastPathBalancedQueueRpcExecutor$FastPathHandler.getCallRunner(FastPathBalancedQueueRpcExecutor.java:109) at org.apache.hadoop.hbase.ipc.RpcExecutor$Handler.run(RpcExecutor.java:339) "RpcServer.default.FPBQ.Fifo.handler=73,queue=3,port=16020" #160 daemon prio=5 os_prio=0 tid=0x00007fc11d885000 nid=0x7415 waiting for monitor entry [0x00007fc1131c3000] java.lang.Thread.State: BLOCKED (on object monitor) at org.apache.hadoop.fs.FSInputStream.read(FSInputStream.java:73) - waiting to lock <0x00007fc4c2769660> (a org.apache.hadoop.fs.s3a.S3AInputStream) at org.apache.hadoop.fs.FSDataInputStream.read(FSDataInputStream.java:97) at org.apache.hadoop.hbase.io.FileLink$FileLinkInputStream.read(FileLink.java:171) at org.apache.hadoop.fs.FSDataInputStream.read(FSDataInputStream.java:97) at org.apache.hadoop.hbase.io.util.BlockIOUtils.preadWithExtra(BlockIOUtils.java:235) at org.apache.hadoop.hbase.io.hfile.HFileBlock$FSReaderImpl.readAtOffset(HFileBlock.java:1564) at org.apache.hadoop.hbase.io.hfile.HFileBlock$FSReaderImpl.readBlockDataInternal(HFileBlock.java:1788) at org.apache.hadoop.hbase.io.hfile.HFileBlock$FSReaderImpl.readBlockData(HFileBlock.java:1597) at org.apache.hadoop.hbase.io.hfile.HFileReaderImpl.readBlock(HFileReaderImpl.java:1347) at org.apache.hadoop.hbase.io.hfile.HFileReaderImpl.readBlock(HFileReaderImpl.java:1268) at org.apache.hadoop.hbase.io.hfile.HFileReaderImpl$HFileScannerImpl.readNextDataBlock(HFileReaderImpl.java:762) at org.apache.hadoop.hbase.io.hfile.HFileReaderImpl$EncodedScanner.next(HFileReaderImpl.java:1540) at org.apache.hadoop.hbase.regionserver.StoreFileScanner.reseekAtOrAfter(StoreFileScanner.java:351) at org.apache.hadoop.hbase.regionserver.StoreFileScanner.reseek(StoreFileScanner.java:244) at org.apache.hadoop.hbase.regionserver.NonLazyKeyValueScanner.doRealSeek(NonLazyKeyValueScanner.java:55) at org.apache.hadoop.hbase.regionserver.KeyValueHeap.generalizedSeek(KeyValueHeap.java:322) at org.apache.hadoop.hbase.regionserver.KeyValueHeap.reseek(KeyValueHeap.java:265) at org.apache.hadoop.hbase.regionserver.StoreScanner.reseek(StoreScanner.java:1095) at org.apache.hadoop.hbase.regionserver.StoreScanner.seekAsDirection(StoreScanner.java:1084) at org.apache.hadoop.hbase.regionserver.StoreScanner.seekOrSkipToNextColumn(StoreScanner.java:821) at org.apache.hadoop.hbase.regionserver.StoreScanner.next(StoreScanner.java:684) at org.apache.hadoop.hbase.regionserver.KeyValueHeap.next(KeyValueHeap.java:155) at org.apache.hadoop.hbase.regionserver.HRegion$RegionScannerImpl.populateResult(HRegion.java:7399) at org.apache.hadoop.hbase.regionserver.HRegion$RegionScannerImpl.nextInternal(HRegion.java:7567) at org.apache.hadoop.hbase.regionserver.HRegion$RegionScannerImpl.nextRaw(HRegion.java:7331) at org.apache.hadoop.hbase.regionserver.HRegion$RegionScannerImpl.nextRaw(HRegion.java:7317) at org.apache.phoenix.coprocessor.PagedRegionScanner.next(PagedRegionScanner.java:57) at org.apache.phoenix.coprocessor.PagedRegionScanner.nextRaw(PagedRegionScanner.java:96) at org.apache.phoenix.iterate.RegionScannerFactory$1.nextRaw(RegionScannerFactory.java:211) at org.apache.phoenix.coprocessor.DelegateRegionScanner.nextRaw(DelegateRegionScanner.java:77) at org.apache.phoenix.coprocessor.DelegateRegionScanner.nextRaw(DelegateRegionScanner.java:77) at org.apache.phoenix.coprocessor.BaseScannerRegionObserver$RegionScannerHolder.nextRaw(BaseScannerRegionObserver.java:308) at org.apache.hadoop.hbase.regionserver.RSRpcServices.scan(RSRpcServices.java:3373) - locked <0x00007fc6b139c718> (a org.apache.phoenix.coprocessor.BaseScannerRegionObserver$RegionScannerHolder) at org.apache.hadoop.hbase.regionserver.RSRpcServices.scan(RSRpcServices.java:3637) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45819) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:392) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:140) at org.apache.hadoop.hbase.ipc.RpcExecutor$Handler.run(RpcExecutor.java:359) at org.apache.hadoop.hbase.ipc.RpcExecutor$Handler.run(RpcExecutor.java:339) "RpcServer.default.FPBQ.Fifo.handler=72,queue=2,port=16020" #159 daemon prio=5 os_prio=0 tid=0x00007fc11d883000 nid=0x7414 waiting for monitor entry [0x00007fc1132c4000] java.lang.Thread.State: BLOCKED (on object monitor) at org.apache.hadoop.fs.FSInputStream.read(FSInputStream.java:73) - waiting to lock <0x00007fc4c2769660> (a org.apache.hadoop.fs.s3a.S3AInputStream) at org.apache.hadoop.fs.FSDataInputStream.read(FSDataInputStream.java:97) at org.apache.hadoop.hbase.io.FileLink$FileLinkInputStream.read(FileLink.java:171) at org.apache.hadoop.fs.FSDataInputStream.read(FSDataInputStream.java:97) at org.apache.hadoop.hbase.io.util.BlockIOUtils.preadWithExtra(BlockIOUtils.java:235) at org.apache.hadoop.hbase.io.hfile.HFileBlock$FSReaderImpl.readAtOffset(HFileBlock.java:1564) at org.apache.hadoop.hbase.io.hfile.HFileBlock$FSReaderImpl.readBlockDataInternal(HFileBlock.java:1788) at org.apache.hadoop.hbase.io.hfile.HFileBlock$FSReaderImpl.readBlockData(HFileBlock.java:1597) at org.apache.hadoop.hbase.io.hfile.HFileReaderImpl.readBlock(HFileReaderImpl.java:1347) at org.apache.hadoop.hbase.io.hfile.HFileReaderImpl.readBlock(HFileReaderImpl.java:1268) at org.apache.hadoop.hbase.io.hfile.HFileReaderImpl$HFileScannerImpl.readNextDataBlock(HFileReaderImpl.java:762) at org.apache.hadoop.hbase.io.hfile.HFileReaderImpl$EncodedScanner.next(HFileReaderImpl.java:1540) at org.apache.hadoop.hbase.regionserver.StoreFileScanner.reseekAtOrAfter(StoreFileScanner.java:351) at org.apache.hadoop.hbase.regionserver.StoreFileScanner.reseek(StoreFileScanner.java:244) at org.apache.hadoop.hbase.regionserver.NonLazyKeyValueScanner.doRealSeek(NonLazyKeyValueScanner.java:55) at org.apache.hadoop.hbase.regionserver.KeyValueHeap.generalizedSeek(KeyValueHeap.java:322) at org.apache.hadoop.hbase.regionserver.KeyValueHeap.reseek(KeyValueHeap.java:265) at org.apache.hadoop.hbase.regionserver.StoreScanner.reseek(StoreScanner.java:1095) at org.apache.hadoop.hbase.regionserver.StoreScanner.seekAsDirection(StoreScanner.java:1084) at org.apache.hadoop.hbase.regionserver.StoreScanner.seekOrSkipToNextColumn(StoreScanner.java:821) at org.apache.hadoop.hbase.regionserver.StoreScanner.next(StoreScanner.java:684) at org.apache.hadoop.hbase.regionserver.KeyValueHeap.next(KeyValueHeap.java:155) at org.apache.hadoop.hbase.regionserver.HRegion$RegionScannerImpl.populateResult(HRegion.java:7399) at org.apache.hadoop.hbase.regionserver.HRegion$RegionScannerImpl.nextInternal(HRegion.java:7567) at org.apache.hadoop.hbase.regionserver.HRegion$RegionScannerImpl.nextRaw(HRegion.java:7331) at org.apache.hadoop.hbase.regionserver.HRegion$RegionScannerImpl.nextRaw(HRegion.java:7317) at org.apache.phoenix.coprocessor.PagedRegionScanner.next(PagedRegionScanner.java:57) at org.apache.phoenix.coprocessor.PagedRegionScanner.nextRaw(PagedRegionScanner.java:96) at org.apache.phoenix.iterate.RegionScannerFactory$1.nextRaw(RegionScannerFactory.java:211) at org.apache.phoenix.coprocessor.DelegateRegionScanner.nextRaw(DelegateRegionScanner.java:77) at org.apache.phoenix.coprocessor.DelegateRegionScanner.nextRaw(DelegateRegionScanner.java:77) at org.apache.phoenix.coprocessor.BaseScannerRegionObserver$RegionScannerHolder.nextRaw(BaseScannerRegionObserver.java:308) at org.apache.hadoop.hbase.regionserver.RSRpcServices.scan(RSRpcServices.java:3373) - locked <0x00007fc85f01ded0> (a org.apache.phoenix.coprocessor.BaseScannerRegionObserver$RegionScannerHolder) at org.apache.hadoop.hbase.regionserver.RSRpcServices.scan(RSRpcServices.java:3637) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45819) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:392) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:140) at org.apache.hadoop.hbase.ipc.RpcExecutor$Handler.run(RpcExecutor.java:359) at org.apache.hadoop.hbase.ipc.RpcExecutor$Handler.run(RpcExecutor.java:339) "RpcServer.default.FPBQ.Fifo.handler=71,queue=1,port=16020" #158 daemon prio=5 os_prio=0 tid=0x00007fc11d881000 nid=0x7413 waiting for monitor entry [0x00007fc1133c5000] java.lang.Thread.State: BLOCKED (on object monitor) at org.apache.hadoop.fs.FSInputStream.read(FSInputStream.java:73) - waiting to lock <0x00007fc4c2769660> (a org.apache.hadoop.fs.s3a.S3AInputStream) at org.apache.hadoop.fs.FSDataInputStream.read(FSDataInputStream.java:97) at org.apache.hadoop.hbase.io.FileLink$FileLinkInputStream.read(FileLink.java:171) at org.apache.hadoop.fs.FSDataInputStream.read(FSDataInputStream.java:97) at org.apache.hadoop.hbase.io.util.BlockIOUtils.preadWithExtra(BlockIOUtils.java:235) at org.apache.hadoop.hbase.io.hfile.HFileBlock$FSReaderImpl.readAtOffset(HFileBlock.java:1564) at org.apache.hadoop.hbase.io.hfile.HFileBlock$FSReaderImpl.readBlockDataInternal(HFileBlock.java:1788) at org.apache.hadoop.hbase.io.hfile.HFileBlock$FSReaderImpl.readBlockData(HFileBlock.java:1597) at org.apache.hadoop.hbase.io.hfile.HFileReaderImpl.readBlock(HFileReaderImpl.java:1347) at org.apache.hadoop.hbase.io.hfile.HFileReaderImpl.readBlock(HFileReaderImpl.java:1268) at org.apache.hadoop.hbase.io.hfile.HFileReaderImpl$HFileScannerImpl.readNextDataBlock(HFileReaderImpl.java:762) at org.apache.hadoop.hbase.io.hfile.HFileReaderImpl$EncodedScanner.next(HFileReaderImpl.java:1540) at org.apache.hadoop.hbase.regionserver.StoreFileScanner.reseekAtOrAfter(StoreFileScanner.java:351) at org.apache.hadoop.hbase.regionserver.StoreFileScanner.reseek(StoreFileScanner.java:244) at org.apache.hadoop.hbase.regionserver.NonLazyKeyValueScanner.doRealSeek(NonLazyKeyValueScanner.java:55) at org.apache.hadoop.hbase.regionserver.KeyValueHeap.generalizedSeek(KeyValueHeap.java:322) at org.apache.hadoop.hbase.regionserver.KeyValueHeap.reseek(KeyValueHeap.java:265) at org.apache.hadoop.hbase.regionserver.StoreScanner.reseek(StoreScanner.java:1095) at org.apache.hadoop.hbase.regionserver.StoreScanner.seekAsDirection(StoreScanner.java:1084) at org.apache.hadoop.hbase.regionserver.StoreScanner.seekOrSkipToNextColumn(StoreScanner.java:821) at org.apache.hadoop.hbase.regionserver.StoreScanner.next(StoreScanner.java:684) at org.apache.hadoop.hbase.regionserver.KeyValueHeap.next(KeyValueHeap.java:155) at org.apache.hadoop.hbase.regionserver.HRegion$RegionScannerImpl.populateResult(HRegion.java:7399) at org.apache.hadoop.hbase.regionserver.HRegion$RegionScannerImpl.nextInternal(HRegion.java:7567) at org.apache.hadoop.hbase.regionserver.HRegion$RegionScannerImpl.nextRaw(HRegion.java:7331) at org.apache.hadoop.hbase.regionserver.HRegion$RegionScannerImpl.nextRaw(HRegion.java:7317) at org.apache.phoenix.coprocessor.PagedRegionScanner.next(PagedRegionScanner.java:57) at org.apache.phoenix.coprocessor.PagedRegionScanner.nextRaw(PagedRegionScanner.java:96) at org.apache.phoenix.iterate.RegionScannerFactory$1.nextRaw(RegionScannerFactory.java:211) at org.apache.phoenix.coprocessor.DelegateRegionScanner.nextRaw(DelegateRegionScanner.java:77) at org.apache.phoenix.coprocessor.DelegateRegionScanner.nextRaw(DelegateRegionScanner.java:77) at org.apache.phoenix.coprocessor.BaseScannerRegionObserver$RegionScannerHolder.nextRaw(BaseScannerRegionObserver.java:308) at org.apache.hadoop.hbase.regionserver.RSRpcServices.scan(RSRpcServices.java:3373) - locked <0x00007fc6b139e320> (a org.apache.phoenix.coprocessor.BaseScannerRegionObserver$RegionScannerHolder) at org.apache.hadoop.hbase.regionserver.RSRpcServices.scan(RSRpcServices.java:3637) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45819) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:392) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:140) at org.apache.hadoop.hbase.ipc.RpcExecutor$Handler.run(RpcExecutor.java:359) at org.apache.hadoop.hbase.ipc.RpcExecutor$Handler.run(RpcExecutor.java:339) "RpcServer.default.FPBQ.Fifo.handler=70,queue=0,port=16020" #157 daemon prio=5 os_prio=0 tid=0x00007fc11d87f000 nid=0x7412 waiting for monitor entry [0x00007fc1134c6000] java.lang.Thread.State: BLOCKED (on object monitor) at org.apache.hadoop.fs.FSInputStream.read(FSInputStream.java:73) - waiting to lock <0x00007fc4c2769660> (a org.apache.hadoop.fs.s3a.S3AInputStream) at org.apache.hadoop.fs.FSDataInputStream.read(FSDataInputStream.java:97) at org.apache.hadoop.hbase.io.FileLink$FileLinkInputStream.read(FileLink.java:171) at org.apache.hadoop.fs.FSDataInputStream.read(FSDataInputStream.java:97) at org.apache.hadoop.hbase.io.util.BlockIOUtils.preadWithExtra(BlockIOUtils.java:235) at org.apache.hadoop.hbase.io.hfile.HFileBlock$FSReaderImpl.readAtOffset(HFileBlock.java:1564) at org.apache.hadoop.hbase.io.hfile.HFileBlock$FSReaderImpl.readBlockDataInternal(HFileBlock.java:1788) at org.apache.hadoop.hbase.io.hfile.HFileBlock$FSReaderImpl.readBlockData(HFileBlock.java:1597) at org.apache.hadoop.hbase.io.hfile.HFileReaderImpl.readBlock(HFileReaderImpl.java:1347) at org.apache.hadoop.hbase.io.hfile.HFileReaderImpl.readBlock(HFileReaderImpl.java:1268) at org.apache.hadoop.hbase.io.hfile.HFileReaderImpl$HFileScannerImpl.readNextDataBlock(HFileReaderImpl.java:762) at org.apache.hadoop.hbase.io.hfile.HFileReaderImpl$EncodedScanner.next(HFileReaderImpl.java:1540) at org.apache.hadoop.hbase.regionserver.StoreFileScanner.reseekAtOrAfter(StoreFileScanner.java:351) at org.apache.hadoop.hbase.regionserver.StoreFileScanner.reseek(StoreFileScanner.java:244) at org.apache.hadoop.hbase.regionserver.NonLazyKeyValueScanner.doRealSeek(NonLazyKeyValueScanner.java:55) at org.apache.hadoop.hbase.regionserver.KeyValueHeap.generalizedSeek(KeyValueHeap.java:322) at org.apache.hadoop.hbase.regionserver.KeyValueHeap.reseek(KeyValueHeap.java:265) at org.apache.hadoop.hbase.regionserver.StoreScanner.reseek(StoreScanner.java:1095) at org.apache.hadoop.hbase.regionserver.StoreScanner.seekAsDirection(StoreScanner.java:1084) at org.apache.hadoop.hbase.regionserver.StoreScanner.seekOrSkipToNextColumn(StoreScanner.java:821) at org.apache.hadoop.hbase.regionserver.StoreScanner.next(StoreScanner.java:684) at org.apache.hadoop.hbase.regionserver.KeyValueHeap.next(KeyValueHeap.java:155) at org.apache.hadoop.hbase.regionserver.HRegion$RegionScannerImpl.populateResult(HRegion.java:7399) at org.apache.hadoop.hbase.regionserver.HRegion$RegionScannerImpl.nextInternal(HRegion.java:7567) at org.apache.hadoop.hbase.regionserver.HRegion$RegionScannerImpl.nextRaw(HRegion.java:7331) at org.apache.hadoop.hbase.regionserver.HRegion$RegionScannerImpl.nextRaw(HRegion.java:7317) at org.apache.phoenix.coprocessor.PagedRegionScanner.next(PagedRegionScanner.java:57) at org.apache.phoenix.coprocessor.PagedRegionScanner.nextRaw(PagedRegionScanner.java:96) at org.apache.phoenix.iterate.RegionScannerFactory$1.nextRaw(RegionScannerFactory.java:211) at org.apache.phoenix.coprocessor.DelegateRegionScanner.nextRaw(DelegateRegionScanner.java:77) at org.apache.phoenix.coprocessor.DelegateRegionScanner.nextRaw(DelegateRegionScanner.java:77) at org.apache.phoenix.coprocessor.BaseScannerRegionObserver$RegionScannerHolder.nextRaw(BaseScannerRegionObserver.java:308) at org.apache.hadoop.hbase.regionserver.RSRpcServices.scan(RSRpcServices.java:3373) - locked <0x00007fc85f00f350> (a org.apache.phoenix.coprocessor.BaseScannerRegionObserver$RegionScannerHolder) at org.apache.hadoop.hbase.regionserver.RSRpcServices.scan(RSRpcServices.java:3637) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45819) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:392) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:140) at org.apache.hadoop.hbase.ipc.RpcExecutor$Handler.run(RpcExecutor.java:359) at org.apache.hadoop.hbase.ipc.RpcExecutor$Handler.run(RpcExecutor.java:339) "RpcServer.default.FPBQ.Fifo.handler=69,queue=9,port=16020" #156 daemon prio=5 os_prio=0 tid=0x00007fc11d87c800 nid=0x7411 waiting for monitor entry [0x00007fc1135c7000] java.lang.Thread.State: BLOCKED (on object monitor) at org.apache.hadoop.fs.FSInputStream.read(FSInputStream.java:73) - waiting to lock <0x00007fc4c2769660> (a org.apache.hadoop.fs.s3a.S3AInputStream) at org.apache.hadoop.fs.FSDataInputStream.read(FSDataInputStream.java:97) at org.apache.hadoop.hbase.io.FileLink$FileLinkInputStream.read(FileLink.java:171) at org.apache.hadoop.fs.FSDataInputStream.read(FSDataInputStream.java:97) at org.apache.hadoop.hbase.io.util.BlockIOUtils.preadWithExtra(BlockIOUtils.java:235) at org.apache.hadoop.hbase.io.hfile.HFileBlock$FSReaderImpl.readAtOffset(HFileBlock.java:1564) at org.apache.hadoop.hbase.io.hfile.HFileBlock$FSReaderImpl.readBlockDataInternal(HFileBlock.java:1788) at org.apache.hadoop.hbase.io.hfile.HFileBlock$FSReaderImpl.readBlockData(HFileBlock.java:1597) at org.apache.hadoop.hbase.io.hfile.HFileReaderImpl.readBlock(HFileReaderImpl.java:1347) at org.apache.hadoop.hbase.io.hfile.HFileReaderImpl.readBlock(HFileReaderImpl.java:1268) at org.apache.hadoop.hbase.io.hfile.HFileReaderImpl$HFileScannerImpl.readNextDataBlock(HFileReaderImpl.java:762) at org.apache.hadoop.hbase.io.hfile.HFileReaderImpl$EncodedScanner.next(HFileReaderImpl.java:1540) at org.apache.hadoop.hbase.regionserver.StoreFileScanner.reseekAtOrAfter(StoreFileScanner.java:351) at org.apache.hadoop.hbase.regionserver.StoreFileScanner.reseek(StoreFileScanner.java:244) at org.apache.hadoop.hbase.regionserver.NonLazyKeyValueScanner.doRealSeek(NonLazyKeyValueScanner.java:55) at org.apache.hadoop.hbase.regionserver.KeyValueHeap.generalizedSeek(KeyValueHeap.java:322) at org.apache.hadoop.hbase.regionserver.KeyValueHeap.reseek(KeyValueHeap.java:265) at org.apache.hadoop.hbase.regionserver.StoreScanner.reseek(StoreScanner.java:1095) at org.apache.hadoop.hbase.regionserver.StoreScanner.seekAsDirection(StoreScanner.java:1084) at org.apache.hadoop.hbase.regionserver.StoreScanner.seekOrSkipToNextColumn(StoreScanner.java:821) at org.apache.hadoop.hbase.regionserver.StoreScanner.next(StoreScanner.java:684) at org.apache.hadoop.hbase.regionserver.KeyValueHeap.next(KeyValueHeap.java:155) at org.apache.hadoop.hbase.regionserver.HRegion$RegionScannerImpl.populateResult(HRegion.java:7399) at org.apache.hadoop.hbase.regionserver.HRegion$RegionScannerImpl.nextInternal(HRegion.java:7567) at org.apache.hadoop.hbase.regionserver.HRegion$RegionScannerImpl.nextRaw(HRegion.java:7331) at org.apache.hadoop.hbase.regionserver.HRegion$RegionScannerImpl.nextRaw(HRegion.java:7317) at org.apache.phoenix.coprocessor.PagedRegionScanner.next(PagedRegionScanner.java:57) at org.apache.phoenix.coprocessor.PagedRegionScanner.nextRaw(PagedRegionScanner.java:96) at org.apache.phoenix.iterate.RegionScannerFactory$1.nextRaw(RegionScannerFactory.java:211) at org.apache.phoenix.coprocessor.DelegateRegionScanner.nextRaw(DelegateRegionScanner.java:77) at org.apache.phoenix.coprocessor.DelegateRegionScanner.nextRaw(DelegateRegionScanner.java:77) at org.apache.phoenix.coprocessor.BaseScannerRegionObserver$RegionScannerHolder.nextRaw(BaseScannerRegionObserver.java:308) at org.apache.hadoop.hbase.regionserver.RSRpcServices.scan(RSRpcServices.java:3373) - locked <0x00007fc85f042700> (a org.apache.phoenix.coprocessor.BaseScannerRegionObserver$RegionScannerHolder) at org.apache.hadoop.hbase.regionserver.RSRpcServices.scan(RSRpcServices.java:3637) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45819) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:392) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:140) at org.apache.hadoop.hbase.ipc.RpcExecutor$Handler.run(RpcExecutor.java:359) at org.apache.hadoop.hbase.ipc.RpcExecutor$Handler.run(RpcExecutor.java:339) "RpcServer.default.FPBQ.Fifo.handler=68,queue=8,port=16020" #155 daemon prio=5 os_prio=0 tid=0x00007fc11d87a800 nid=0x7410 waiting for monitor entry [0x00007fc1136c8000] java.lang.Thread.State: BLOCKED (on object monitor) at org.apache.hadoop.fs.FSInputStream.read(FSInputStream.java:73) - waiting to lock <0x00007fc4c2769660> (a org.apache.hadoop.fs.s3a.S3AInputStream) at org.apache.hadoop.fs.FSDataInputStream.read(FSDataInputStream.java:97) at org.apache.hadoop.hbase.io.FileLink$FileLinkInputStream.read(FileLink.java:171) at org.apache.hadoop.fs.FSDataInputStream.read(FSDataInputStream.java:97) at org.apache.hadoop.hbase.io.util.BlockIOUtils.preadWithExtra(BlockIOUtils.java:235) at org.apache.hadoop.hbase.io.hfile.HFileBlock$FSReaderImpl.readAtOffset(HFileBlock.java:1564) at org.apache.hadoop.hbase.io.hfile.HFileBlock$FSReaderImpl.readBlockDataInternal(HFileBlock.java:1788) at org.apache.hadoop.hbase.io.hfile.HFileBlock$FSReaderImpl.readBlockData(HFileBlock.java:1597) at org.apache.hadoop.hbase.io.hfile.HFileReaderImpl.readBlock(HFileReaderImpl.java:1347) at org.apache.hadoop.hbase.io.hfile.HFileReaderImpl.readBlock(HFileReaderImpl.java:1268) at org.apache.hadoop.hbase.io.hfile.HFileReaderImpl$HFileScannerImpl.readNextDataBlock(HFileReaderImpl.java:762) at org.apache.hadoop.hbase.io.hfile.HFileReaderImpl$EncodedScanner.next(HFileReaderImpl.java:1540) at org.apache.hadoop.hbase.regionserver.StoreFileScanner.reseekAtOrAfter(StoreFileScanner.java:351) at org.apache.hadoop.hbase.regionserver.StoreFileScanner.reseek(StoreFileScanner.java:244) at org.apache.hadoop.hbase.regionserver.NonLazyKeyValueScanner.doRealSeek(NonLazyKeyValueScanner.java:55) at org.apache.hadoop.hbase.regionserver.KeyValueHeap.generalizedSeek(KeyValueHeap.java:322) at org.apache.hadoop.hbase.regionserver.KeyValueHeap.reseek(KeyValueHeap.java:265) at org.apache.hadoop.hbase.regionserver.StoreScanner.reseek(StoreScanner.java:1095) at org.apache.hadoop.hbase.regionserver.StoreScanner.seekAsDirection(StoreScanner.java:1084) at org.apache.hadoop.hbase.regionserver.StoreScanner.seekOrSkipToNextColumn(StoreScanner.java:821) at org.apache.hadoop.hbase.regionserver.StoreScanner.next(StoreScanner.java:684) at org.apache.hadoop.hbase.regionserver.KeyValueHeap.next(KeyValueHeap.java:155) at org.apache.hadoop.hbase.regionserver.HRegion$RegionScannerImpl.populateResult(HRegion.java:7399) at org.apache.hadoop.hbase.regionserver.HRegion$RegionScannerImpl.nextInternal(HRegion.java:7567) at org.apache.hadoop.hbase.regionserver.HRegion$RegionScannerImpl.nextRaw(HRegion.java:7331) at org.apache.hadoop.hbase.regionserver.HRegion$RegionScannerImpl.nextRaw(HRegion.java:7317) at org.apache.phoenix.coprocessor.PagedRegionScanner.next(PagedRegionScanner.java:57) at org.apache.phoenix.coprocessor.PagedRegionScanner.nextRaw(PagedRegionScanner.java:96) at org.apache.phoenix.iterate.RegionScannerFactory$1.nextRaw(RegionScannerFactory.java:211) at org.apache.phoenix.coprocessor.DelegateRegionScanner.nextRaw(DelegateRegionScanner.java:77) at org.apache.phoenix.coprocessor.DelegateRegionScanner.nextRaw(DelegateRegionScanner.java:77) at org.apache.phoenix.coprocessor.BaseScannerRegionObserver$RegionScannerHolder.nextRaw(BaseScannerRegionObserver.java:308) at org.apache.hadoop.hbase.regionserver.RSRpcServices.scan(RSRpcServices.java:3373) - locked <0x00007fc85f01ff88> (a org.apache.phoenix.coprocessor.BaseScannerRegionObserver$RegionScannerHolder) at org.apache.hadoop.hbase.regionserver.RSRpcServices.scan(RSRpcServices.java:3637) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45819) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:392) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:140) at org.apache.hadoop.hbase.ipc.RpcExecutor$Handler.run(RpcExecutor.java:359) at org.apache.hadoop.hbase.ipc.RpcExecutor$Handler.run(RpcExecutor.java:339) "RpcServer.default.FPBQ.Fifo.handler=67,queue=7,port=16020" #154 daemon prio=5 os_prio=0 tid=0x00007fc11d878800 nid=0x740f waiting for monitor entry [0x00007fc1137c9000] java.lang.Thread.State: BLOCKED (on object monitor) at org.apache.hadoop.fs.FSInputStream.read(FSInputStream.java:73) - waiting to lock <0x00007fc4c2769660> (a org.apache.hadoop.fs.s3a.S3AInputStream) at org.apache.hadoop.fs.FSDataInputStream.read(FSDataInputStream.java:97) at org.apache.hadoop.hbase.io.FileLink$FileLinkInputStream.read(FileLink.java:171) at org.apache.hadoop.fs.FSDataInputStream.read(FSDataInputStream.java:97) at org.apache.hadoop.hbase.io.util.BlockIOUtils.preadWithExtra(BlockIOUtils.java:235) at org.apache.hadoop.hbase.io.hfile.HFileBlock$FSReaderImpl.readAtOffset(HFileBlock.java:1564) at org.apache.hadoop.hbase.io.hfile.HFileBlock$FSReaderImpl.readBlockDataInternal(HFileBlock.java:1788) at org.apache.hadoop.hbase.io.hfile.HFileBlock$FSReaderImpl.readBlockData(HFileBlock.java:1597) at org.apache.hadoop.hbase.io.hfile.HFileReaderImpl.readBlock(HFileReaderImpl.java:1347) at org.apache.hadoop.hbase.io.hfile.HFileReaderImpl.readBlock(HFileReaderImpl.java:1268) at org.apache.hadoop.hbase.io.hfile.HFileReaderImpl$HFileScannerImpl.readNextDataBlock(HFileReaderImpl.java:762) at org.apache.hadoop.hbase.io.hfile.HFileReaderImpl$EncodedScanner.next(HFileReaderImpl.java:1540) at org.apache.hadoop.hbase.regionserver.StoreFileScanner.reseekAtOrAfter(StoreFileScanner.java:351) at org.apache.hadoop.hbase.regionserver.StoreFileScanner.reseek(StoreFileScanner.java:244) at org.apache.hadoop.hbase.regionserver.NonLazyKeyValueScanner.doRealSeek(NonLazyKeyValueScanner.java:55) at org.apache.hadoop.hbase.regionserver.KeyValueHeap.generalizedSeek(KeyValueHeap.java:322) at org.apache.hadoop.hbase.regionserver.KeyValueHeap.reseek(KeyValueHeap.java:265) at org.apache.hadoop.hbase.regionserver.StoreScanner.reseek(StoreScanner.java:1095) at org.apache.hadoop.hbase.regionserver.StoreScanner.seekAsDirection(StoreScanner.java:1084) at org.apache.hadoop.hbase.regionserver.StoreScanner.seekOrSkipToNextColumn(StoreScanner.java:821) at org.apache.hadoop.hbase.regionserver.StoreScanner.next(StoreScanner.java:684) at org.apache.hadoop.hbase.regionserver.KeyValueHeap.next(KeyValueHeap.java:155) at org.apache.hadoop.hbase.regionserver.HRegion$RegionScannerImpl.populateResult(HRegion.java:7399) at org.apache.hadoop.hbase.regionserver.HRegion$RegionScannerImpl.nextInternal(HRegion.java:7567) at org.apache.hadoop.hbase.regionserver.HRegion$RegionScannerImpl.nextRaw(HRegion.java:7331) at org.apache.hadoop.hbase.regionserver.HRegion$RegionScannerImpl.nextRaw(HRegion.java:7317) at org.apache.phoenix.coprocessor.PagedRegionScanner.next(PagedRegionScanner.java:57) at org.apache.phoenix.coprocessor.PagedRegionScanner.nextRaw(PagedRegionScanner.java:96) at org.apache.phoenix.iterate.RegionScannerFactory$1.nextRaw(RegionScannerFactory.java:211) at org.apache.phoenix.coprocessor.DelegateRegionScanner.nextRaw(DelegateRegionScanner.java:77) at org.apache.phoenix.coprocessor.DelegateRegionScanner.nextRaw(DelegateRegionScanner.java:77) at org.apache.phoenix.coprocessor.BaseScannerRegionObserver$RegionScannerHolder.nextRaw(BaseScannerRegionObserver.java:308) at org.apache.hadoop.hbase.regionserver.RSRpcServices.scan(RSRpcServices.java:3373) - locked <0x00007fc85f00f9e0> (a org.apache.phoenix.coprocessor.BaseScannerRegionObserver$RegionScannerHolder) at org.apache.hadoop.hbase.regionserver.RSRpcServices.scan(RSRpcServices.java:3637) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45819) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:392) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:140) at org.apache.hadoop.hbase.ipc.RpcExecutor$Handler.run(RpcExecutor.java:359) at org.apache.hadoop.hbase.ipc.RpcExecutor$Handler.run(RpcExecutor.java:339) "RpcServer.default.FPBQ.Fifo.handler=66,queue=6,port=16020" #153 daemon prio=5 os_prio=0 tid=0x00007fc11d876800 nid=0x740e waiting for monitor entry [0x00007fc1138ca000] java.lang.Thread.State: BLOCKED (on object monitor) at org.apache.hadoop.fs.FSInputStream.read(FSInputStream.java:73) - waiting to lock <0x00007fc4c2769660> (a org.apache.hadoop.fs.s3a.S3AInputStream) at org.apache.hadoop.fs.FSDataInputStream.read(FSDataInputStream.java:97) at org.apache.hadoop.hbase.io.FileLink$FileLinkInputStream.read(FileLink.java:171) at org.apache.hadoop.fs.FSDataInputStream.read(FSDataInputStream.java:97) at org.apache.hadoop.hbase.io.util.BlockIOUtils.preadWithExtra(BlockIOUtils.java:235) at org.apache.hadoop.hbase.io.hfile.HFileBlock$FSReaderImpl.readAtOffset(HFileBlock.java:1564) at org.apache.hadoop.hbase.io.hfile.HFileBlock$FSReaderImpl.readBlockDataInternal(HFileBlock.java:1788) at org.apache.hadoop.hbase.io.hfile.HFileBlock$FSReaderImpl.readBlockData(HFileBlock.java:1597) at org.apache.hadoop.hbase.io.hfile.HFileReaderImpl.readBlock(HFileReaderImpl.java:1347) at org.apache.hadoop.hbase.io.hfile.HFileReaderImpl.readBlock(HFileReaderImpl.java:1268) at org.apache.hadoop.hbase.io.hfile.HFileReaderImpl$HFileScannerImpl.readNextDataBlock(HFileReaderImpl.java:762) at org.apache.hadoop.hbase.io.hfile.HFileReaderImpl$EncodedScanner.next(HFileReaderImpl.java:1540) at org.apache.hadoop.hbase.regionserver.StoreFileScanner.reseekAtOrAfter(StoreFileScanner.java:351) at org.apache.hadoop.hbase.regionserver.StoreFileScanner.reseek(StoreFileScanner.java:244) at org.apache.hadoop.hbase.regionserver.NonLazyKeyValueScanner.doRealSeek(NonLazyKeyValueScanner.java:55) at org.apache.hadoop.hbase.regionserver.KeyValueHeap.generalizedSeek(KeyValueHeap.java:322) at org.apache.hadoop.hbase.regionserver.KeyValueHeap.reseek(KeyValueHeap.java:265) at org.apache.hadoop.hbase.regionserver.StoreScanner.reseek(StoreScanner.java:1095) at org.apache.hadoop.hbase.regionserver.StoreScanner.seekAsDirection(StoreScanner.java:1084) at org.apache.hadoop.hbase.regionserver.StoreScanner.seekOrSkipToNextColumn(StoreScanner.java:821) at org.apache.hadoop.hbase.regionserver.StoreScanner.next(StoreScanner.java:684) at org.apache.hadoop.hbase.regionserver.KeyValueHeap.next(KeyValueHeap.java:155) at org.apache.hadoop.hbase.regionserver.HRegion$RegionScannerImpl.populateResult(HRegion.java:7399) at org.apache.hadoop.hbase.regionserver.HRegion$RegionScannerImpl.nextInternal(HRegion.java:7567) at org.apache.hadoop.hbase.regionserver.HRegion$RegionScannerImpl.nextRaw(HRegion.java:7331) at org.apache.hadoop.hbase.regionserver.HRegion$RegionScannerImpl.nextRaw(HRegion.java:7317) at org.apache.phoenix.coprocessor.PagedRegionScanner.next(PagedRegionScanner.java:57) at org.apache.phoenix.coprocessor.PagedRegionScanner.nextRaw(PagedRegionScanner.java:96) at org.apache.phoenix.iterate.RegionScannerFactory$1.nextRaw(RegionScannerFactory.java:211) at org.apache.phoenix.coprocessor.DelegateRegionScanner.nextRaw(DelegateRegionScanner.java:77) at org.apache.phoenix.coprocessor.DelegateRegionScanner.nextRaw(DelegateRegionScanner.java:77) at org.apache.phoenix.coprocessor.BaseScannerRegionObserver$RegionScannerHolder.nextRaw(BaseScannerRegionObserver.java:308) at org.apache.hadoop.hbase.regionserver.RSRpcServices.scan(RSRpcServices.java:3373) - locked <0x00007fc6b139e7a0> (a org.apache.phoenix.coprocessor.BaseScannerRegionObserver$RegionScannerHolder) at org.apache.hadoop.hbase.regionserver.RSRpcServices.scan(RSRpcServices.java:3637) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45819) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:392) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:140) at org.apache.hadoop.hbase.ipc.RpcExecutor$Handler.run(RpcExecutor.java:359) at org.apache.hadoop.hbase.ipc.RpcExecutor$Handler.run(RpcExecutor.java:339) "RpcServer.default.FPBQ.Fifo.handler=65,queue=5,port=16020" #152 daemon prio=5 os_prio=0 tid=0x00007fc11d874000 nid=0x740d waiting for monitor entry [0x00007fc1139cb000] java.lang.Thread.State: BLOCKED (on object monitor) at org.apache.hadoop.fs.FSInputStream.read(FSInputStream.java:73) - waiting to lock <0x00007fc4c2769660> (a org.apache.hadoop.fs.s3a.S3AInputStream) at org.apache.hadoop.fs.FSDataInputStream.read(FSDataInputStream.java:97) at org.apache.hadoop.hbase.io.FileLink$FileLinkInputStream.read(FileLink.java:171) at org.apache.hadoop.fs.FSDataInputStream.read(FSDataInputStream.java:97) at org.apache.hadoop.hbase.io.util.BlockIOUtils.preadWithExtra(BlockIOUtils.java:235) at org.apache.hadoop.hbase.io.hfile.HFileBlock$FSReaderImpl.readAtOffset(HFileBlock.java:1564) at org.apache.hadoop.hbase.io.hfile.HFileBlock$FSReaderImpl.readBlockDataInternal(HFileBlock.java:1788) at org.apache.hadoop.hbase.io.hfile.HFileBlock$FSReaderImpl.readBlockData(HFileBlock.java:1597) at org.apache.hadoop.hbase.io.hfile.HFileReaderImpl.readBlock(HFileReaderImpl.java:1347) at org.apache.hadoop.hbase.io.hfile.HFileReaderImpl.readBlock(HFileReaderImpl.java:1268) at org.apache.hadoop.hbase.io.hfile.HFileReaderImpl$HFileScannerImpl.readNextDataBlock(HFileReaderImpl.java:762) at org.apache.hadoop.hbase.io.hfile.HFileReaderImpl$EncodedScanner.next(HFileReaderImpl.java:1540) at org.apache.hadoop.hbase.regionserver.StoreFileScanner.reseekAtOrAfter(StoreFileScanner.java:351) at org.apache.hadoop.hbase.regionserver.StoreFileScanner.reseek(StoreFileScanner.java:244) at org.apache.hadoop.hbase.regionserver.NonLazyKeyValueScanner.doRealSeek(NonLazyKeyValueScanner.java:55) at org.apache.hadoop.hbase.regionserver.KeyValueHeap.generalizedSeek(KeyValueHeap.java:322) at org.apache.hadoop.hbase.regionserver.KeyValueHeap.reseek(KeyValueHeap.java:265) at org.apache.hadoop.hbase.regionserver.StoreScanner.reseek(StoreScanner.java:1095) at org.apache.hadoop.hbase.regionserver.StoreScanner.seekAsDirection(StoreScanner.java:1084) at org.apache.hadoop.hbase.regionserver.StoreScanner.seekOrSkipToNextColumn(StoreScanner.java:821) at org.apache.hadoop.hbase.regionserver.StoreScanner.next(StoreScanner.java:684) at org.apache.hadoop.hbase.regionserver.KeyValueHeap.next(KeyValueHeap.java:155) at org.apache.hadoop.hbase.regionserver.HRegion$RegionScannerImpl.populateResult(HRegion.java:7399) at org.apache.hadoop.hbase.regionserver.HRegion$RegionScannerImpl.nextInternal(HRegion.java:7567) at org.apache.hadoop.hbase.regionserver.HRegion$RegionScannerImpl.nextRaw(HRegion.java:7331) at org.apache.hadoop.hbase.regionserver.HRegion$RegionScannerImpl.nextRaw(HRegion.java:7317) at org.apache.phoenix.coprocessor.PagedRegionScanner.next(PagedRegionScanner.java:57) at org.apache.phoenix.coprocessor.PagedRegionScanner.nextRaw(PagedRegionScanner.java:96) at org.apache.phoenix.iterate.RegionScannerFactory$1.nextRaw(RegionScannerFactory.java:211) at org.apache.phoenix.coprocessor.DelegateRegionScanner.nextRaw(DelegateRegionScanner.java:77) at org.apache.phoenix.coprocessor.DelegateRegionScanner.nextRaw(DelegateRegionScanner.java:77) at org.apache.phoenix.coprocessor.BaseScannerRegionObserver$RegionScannerHolder.nextRaw(BaseScannerRegionObserver.java:308) at org.apache.hadoop.hbase.regionserver.RSRpcServices.scan(RSRpcServices.java:3373) - locked <0x00007fc6b139ebf8> (a org.apache.phoenix.coprocessor.BaseScannerRegionObserver$RegionScannerHolder) at org.apache.hadoop.hbase.regionserver.RSRpcServices.scan(RSRpcServices.java:3637) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45819) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:392) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:140) at org.apache.hadoop.hbase.ipc.RpcExecutor$Handler.run(RpcExecutor.java:359) at org.apache.hadoop.hbase.ipc.RpcExecutor$Handler.run(RpcExecutor.java:339) "RpcServer.default.FPBQ.Fifo.handler=64,queue=4,port=16020" #151 daemon prio=5 os_prio=0 tid=0x00007fc11d872000 nid=0x740c waiting for monitor entry [0x00007fc113acc000] java.lang.Thread.State: BLOCKED (on object monitor) at org.apache.hadoop.fs.FSInputStream.read(FSInputStream.java:73) - waiting to lock <0x00007fc4c2769660> (a org.apache.hadoop.fs.s3a.S3AInputStream) at org.apache.hadoop.fs.FSDataInputStream.read(FSDataInputStream.java:97) at org.apache.hadoop.hbase.io.FileLink$FileLinkInputStream.read(FileLink.java:171) at org.apache.hadoop.fs.FSDataInputStream.read(FSDataInputStream.java:97) at org.apache.hadoop.hbase.io.util.BlockIOUtils.preadWithExtra(BlockIOUtils.java:235) at org.apache.hadoop.hbase.io.hfile.HFileBlock$FSReaderImpl.readAtOffset(HFileBlock.java:1564) at org.apache.hadoop.hbase.io.hfile.HFileBlock$FSReaderImpl.readBlockDataInternal(HFileBlock.java:1788) at org.apache.hadoop.hbase.io.hfile.HFileBlock$FSReaderImpl.readBlockData(HFileBlock.java:1597) at org.apache.hadoop.hbase.io.hfile.HFileReaderImpl.readBlock(HFileReaderImpl.java:1347) at org.apache.hadoop.hbase.io.hfile.HFileReaderImpl.readBlock(HFileReaderImpl.java:1268) at org.apache.hadoop.hbase.io.hfile.HFileReaderImpl$HFileScannerImpl.readNextDataBlock(HFileReaderImpl.java:762) at org.apache.hadoop.hbase.io.hfile.HFileReaderImpl$EncodedScanner.next(HFileReaderImpl.java:1540) at org.apache.hadoop.hbase.regionserver.StoreFileScanner.reseekAtOrAfter(StoreFileScanner.java:351) at org.apache.hadoop.hbase.regionserver.StoreFileScanner.reseek(StoreFileScanner.java:244) at org.apache.hadoop.hbase.regionserver.NonLazyKeyValueScanner.doRealSeek(NonLazyKeyValueScanner.java:55) at org.apache.hadoop.hbase.regionserver.KeyValueHeap.generalizedSeek(KeyValueHeap.java:322) at org.apache.hadoop.hbase.regionserver.KeyValueHeap.reseek(KeyValueHeap.java:265) at org.apache.hadoop.hbase.regionserver.StoreScanner.reseek(StoreScanner.java:1095) at org.apache.hadoop.hbase.regionserver.StoreScanner.seekAsDirection(StoreScanner.java:1084) at org.apache.hadoop.hbase.regionserver.StoreScanner.seekOrSkipToNextColumn(StoreScanner.java:821) at org.apache.hadoop.hbase.regionserver.StoreScanner.next(StoreScanner.java:684) at org.apache.hadoop.hbase.regionserver.KeyValueHeap.next(KeyValueHeap.java:155) at org.apache.hadoop.hbase.regionserver.HRegion$RegionScannerImpl.populateResult(HRegion.java:7399) at org.apache.hadoop.hbase.regionserver.HRegion$RegionScannerImpl.nextInternal(HRegion.java:7567) at org.apache.hadoop.hbase.regionserver.HRegion$RegionScannerImpl.nextRaw(HRegion.java:7331) at org.apache.hadoop.hbase.regionserver.HRegion$RegionScannerImpl.nextRaw(HRegion.java:7317) at org.apache.phoenix.coprocessor.PagedRegionScanner.next(PagedRegionScanner.java:57) at org.apache.phoenix.coprocessor.PagedRegionScanner.nextRaw(PagedRegionScanner.java:96) at org.apache.phoenix.iterate.RegionScannerFactory$1.nextRaw(RegionScannerFactory.java:211) at org.apache.phoenix.coprocessor.DelegateRegionScanner.nextRaw(DelegateRegionScanner.java:77) at org.apache.phoenix.coprocessor.DelegateRegionScanner.nextRaw(DelegateRegionScanner.java:77) at org.apache.phoenix.coprocessor.BaseScannerRegionObserver$RegionScannerHolder.nextRaw(BaseScannerRegionObserver.java:308) at org.apache.hadoop.hbase.regionserver.RSRpcServices.scan(RSRpcServices.java:3373) - locked <0x00007fc85f044888> (a org.apache.phoenix.coprocessor.BaseScannerRegionObserver$RegionScannerHolder) at org.apache.hadoop.hbase.regionserver.RSRpcServices.scan(RSRpcServices.java:3637) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45819) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:392) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:140) at org.apache.hadoop.hbase.ipc.RpcExecutor$Handler.run(RpcExecutor.java:359) at org.apache.hadoop.hbase.ipc.RpcExecutor$Handler.run(RpcExecutor.java:339) "RpcServer.default.FPBQ.Fifo.handler=63,queue=3,port=16020" #150 daemon prio=5 os_prio=0 tid=0x00007fc11d870000 nid=0x740b waiting on condition [0x00007fc113bce000] java.lang.Thread.State: WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007fc1c9fe59b0> (a java.util.concurrent.Semaphore$NonfairSync) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer.parkAndCheckInterrupt(AbstractQueuedSynchronizer.java:836) at java.util.concurrent.locks.AbstractQueuedSynchronizer.doAcquireSharedInterruptibly(AbstractQueuedSynchronizer.java:997) at java.util.concurrent.locks.AbstractQueuedSynchronizer.acquireSharedInterruptibly(AbstractQueuedSynchronizer.java:1304) at java.util.concurrent.Semaphore.acquire(Semaphore.java:312) at org.apache.hadoop.hbase.ipc.FastPathBalancedQueueRpcExecutor$FastPathHandler.getCallRunner(FastPathBalancedQueueRpcExecutor.java:109) at org.apache.hadoop.hbase.ipc.RpcExecutor$Handler.run(RpcExecutor.java:339) "RpcServer.default.FPBQ.Fifo.handler=62,queue=2,port=16020" #149 daemon prio=5 os_prio=0 tid=0x00007fc11d86e000 nid=0x740a waiting on condition [0x00007fc113ccf000] java.lang.Thread.State: WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007fc1c9fea850> (a java.util.concurrent.Semaphore$NonfairSync) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer.parkAndCheckInterrupt(AbstractQueuedSynchronizer.java:836) at java.util.concurrent.locks.AbstractQueuedSynchronizer.doAcquireSharedInterruptibly(AbstractQueuedSynchronizer.java:997) at java.util.concurrent.locks.AbstractQueuedSynchronizer.acquireSharedInterruptibly(AbstractQueuedSynchronizer.java:1304) at java.util.concurrent.Semaphore.acquire(Semaphore.java:312) at org.apache.hadoop.hbase.ipc.FastPathBalancedQueueRpcExecutor$FastPathHandler.getCallRunner(FastPathBalancedQueueRpcExecutor.java:109) at org.apache.hadoop.hbase.ipc.RpcExecutor$Handler.run(RpcExecutor.java:339) "RpcServer.default.FPBQ.Fifo.handler=61,queue=1,port=16020" #148 daemon prio=5 os_prio=0 tid=0x00007fc11d86c000 nid=0x7409 waiting for monitor entry [0x00007fc113dcf000] java.lang.Thread.State: BLOCKED (on object monitor) at org.apache.hadoop.fs.FSInputStream.read(FSInputStream.java:73) - waiting to lock <0x00007fc4c2769660> (a org.apache.hadoop.fs.s3a.S3AInputStream) at org.apache.hadoop.fs.FSDataInputStream.read(FSDataInputStream.java:97) at org.apache.hadoop.hbase.io.FileLink$FileLinkInputStream.read(FileLink.java:171) at org.apache.hadoop.fs.FSDataInputStream.read(FSDataInputStream.java:97) at org.apache.hadoop.hbase.io.util.BlockIOUtils.preadWithExtra(BlockIOUtils.java:235) at org.apache.hadoop.hbase.io.hfile.HFileBlock$FSReaderImpl.readAtOffset(HFileBlock.java:1564) at org.apache.hadoop.hbase.io.hfile.HFileBlock$FSReaderImpl.readBlockDataInternal(HFileBlock.java:1788) at org.apache.hadoop.hbase.io.hfile.HFileBlock$FSReaderImpl.readBlockData(HFileBlock.java:1597) at org.apache.hadoop.hbase.io.hfile.HFileReaderImpl.readBlock(HFileReaderImpl.java:1347) at org.apache.hadoop.hbase.io.hfile.HFileReaderImpl.readBlock(HFileReaderImpl.java:1268) at org.apache.hadoop.hbase.io.hfile.HFileReaderImpl$HFileScannerImpl.readNextDataBlock(HFileReaderImpl.java:762) at org.apache.hadoop.hbase.io.hfile.HFileReaderImpl$EncodedScanner.next(HFileReaderImpl.java:1540) at org.apache.hadoop.hbase.regionserver.StoreFileScanner.reseekAtOrAfter(StoreFileScanner.java:351) at org.apache.hadoop.hbase.regionserver.StoreFileScanner.reseek(StoreFileScanner.java:244) at org.apache.hadoop.hbase.regionserver.NonLazyKeyValueScanner.doRealSeek(NonLazyKeyValueScanner.java:55) at org.apache.hadoop.hbase.regionserver.KeyValueHeap.generalizedSeek(KeyValueHeap.java:322) at org.apache.hadoop.hbase.regionserver.KeyValueHeap.reseek(KeyValueHeap.java:265) at org.apache.hadoop.hbase.regionserver.StoreScanner.reseek(StoreScanner.java:1095) at org.apache.hadoop.hbase.regionserver.StoreScanner.seekAsDirection(StoreScanner.java:1084) at org.apache.hadoop.hbase.regionserver.StoreScanner.seekOrSkipToNextColumn(StoreScanner.java:821) at org.apache.hadoop.hbase.regionserver.StoreScanner.next(StoreScanner.java:684) at org.apache.hadoop.hbase.regionserver.KeyValueHeap.next(KeyValueHeap.java:155) at org.apache.hadoop.hbase.regionserver.HRegion$RegionScannerImpl.populateResult(HRegion.java:7399) at org.apache.hadoop.hbase.regionserver.HRegion$RegionScannerImpl.nextInternal(HRegion.java:7567) at org.apache.hadoop.hbase.regionserver.HRegion$RegionScannerImpl.nextRaw(HRegion.java:7331) at org.apache.hadoop.hbase.regionserver.HRegion$RegionScannerImpl.nextRaw(HRegion.java:7317) at org.apache.phoenix.coprocessor.PagedRegionScanner.next(PagedRegionScanner.java:57) at org.apache.phoenix.coprocessor.PagedRegionScanner.nextRaw(PagedRegionScanner.java:96) at org.apache.phoenix.iterate.RegionScannerFactory$1.nextRaw(RegionScannerFactory.java:211) at org.apache.phoenix.coprocessor.DelegateRegionScanner.nextRaw(DelegateRegionScanner.java:77) at org.apache.phoenix.coprocessor.DelegateRegionScanner.nextRaw(DelegateRegionScanner.java:77) at org.apache.phoenix.coprocessor.BaseScannerRegionObserver$RegionScannerHolder.nextRaw(BaseScannerRegionObserver.java:308) at org.apache.hadoop.hbase.regionserver.RSRpcServices.scan(RSRpcServices.java:3373) - locked <0x00007fc85f0721b8> (a org.apache.phoenix.coprocessor.BaseScannerRegionObserver$RegionScannerHolder) at org.apache.hadoop.hbase.regionserver.RSRpcServices.scan(RSRpcServices.java:3637) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45819) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:392) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:140) at org.apache.hadoop.hbase.ipc.RpcExecutor$Handler.run(RpcExecutor.java:359) at org.apache.hadoop.hbase.ipc.RpcExecutor$Handler.run(RpcExecutor.java:339) "RpcServer.default.FPBQ.Fifo.handler=60,queue=0,port=16020" #147 daemon prio=5 os_prio=0 tid=0x00007fc11d869800 nid=0x7408 waiting for monitor entry [0x00007fc113ed0000] java.lang.Thread.State: BLOCKED (on object monitor) at org.apache.hadoop.fs.FSInputStream.read(FSInputStream.java:73) - waiting to lock <0x00007fc4c2769660> (a org.apache.hadoop.fs.s3a.S3AInputStream) at org.apache.hadoop.fs.FSDataInputStream.read(FSDataInputStream.java:97) at org.apache.hadoop.hbase.io.FileLink$FileLinkInputStream.read(FileLink.java:171) at org.apache.hadoop.fs.FSDataInputStream.read(FSDataInputStream.java:97) at org.apache.hadoop.hbase.io.util.BlockIOUtils.preadWithExtra(BlockIOUtils.java:235) at org.apache.hadoop.hbase.io.hfile.HFileBlock$FSReaderImpl.readAtOffset(HFileBlock.java:1564) at org.apache.hadoop.hbase.io.hfile.HFileBlock$FSReaderImpl.readBlockDataInternal(HFileBlock.java:1788) at org.apache.hadoop.hbase.io.hfile.HFileBlock$FSReaderImpl.readBlockData(HFileBlock.java:1597) at org.apache.hadoop.hbase.io.hfile.HFileReaderImpl.readBlock(HFileReaderImpl.java:1347) at org.apache.hadoop.hbase.io.hfile.HFileReaderImpl.readBlock(HFileReaderImpl.java:1268) at org.apache.hadoop.hbase.io.hfile.HFileBlockIndex$CellBasedKeyBlockIndexReader.loadDataBlockWithScanInfo(HFileBlockIndex.java:331) at org.apache.hadoop.hbase.io.hfile.HFileReaderImpl$HFileScannerImpl.seekTo(HFileReaderImpl.java:680) at org.apache.hadoop.hbase.io.hfile.HFileReaderImpl$HFileScannerImpl.reseekTo(HFileReaderImpl.java:662) at org.apache.hadoop.hbase.regionserver.StoreFileScanner.reseekAtOrAfter(StoreFileScanner.java:335) at org.apache.hadoop.hbase.regionserver.StoreFileScanner.reseek(StoreFileScanner.java:244) at org.apache.hadoop.hbase.regionserver.NonLazyKeyValueScanner.doRealSeek(NonLazyKeyValueScanner.java:55) at org.apache.hadoop.hbase.regionserver.KeyValueHeap.generalizedSeek(KeyValueHeap.java:322) at org.apache.hadoop.hbase.regionserver.KeyValueHeap.reseek(KeyValueHeap.java:265) at org.apache.hadoop.hbase.regionserver.StoreScanner.reseek(StoreScanner.java:1095) at org.apache.hadoop.hbase.regionserver.StoreScanner.seekToNextRow(StoreScanner.java:1073) at org.apache.hadoop.hbase.regionserver.StoreScanner.seekOrSkipToNextRow(StoreScanner.java:816) at org.apache.hadoop.hbase.regionserver.StoreScanner.next(StoreScanner.java:720) at org.apache.hadoop.hbase.regionserver.KeyValueHeap.next(KeyValueHeap.java:155) at org.apache.hadoop.hbase.regionserver.HRegion$RegionScannerImpl.populateResult(HRegion.java:7399) at org.apache.hadoop.hbase.regionserver.HRegion$RegionScannerImpl.nextInternal(HRegion.java:7567) at org.apache.hadoop.hbase.regionserver.HRegion$RegionScannerImpl.nextRaw(HRegion.java:7331) at org.apache.hadoop.hbase.regionserver.HRegion$RegionScannerImpl.nextRaw(HRegion.java:7317) at org.apache.phoenix.coprocessor.PagedRegionScanner.next(PagedRegionScanner.java:57) at org.apache.phoenix.coprocessor.PagedRegionScanner.nextRaw(PagedRegionScanner.java:96) at org.apache.phoenix.iterate.RegionScannerFactory$1.nextRaw(RegionScannerFactory.java:211) at org.apache.phoenix.coprocessor.DelegateRegionScanner.nextRaw(DelegateRegionScanner.java:77) at org.apache.phoenix.coprocessor.DelegateRegionScanner.nextRaw(DelegateRegionScanner.java:77) at org.apache.phoenix.coprocessor.BaseScannerRegionObserver$RegionScannerHolder.nextRaw(BaseScannerRegionObserver.java:308) at org.apache.hadoop.hbase.regionserver.RSRpcServices.scan(RSRpcServices.java:3373) - locked <0x00007fc930974470> (a org.apache.phoenix.coprocessor.BaseScannerRegionObserver$RegionScannerHolder) at org.apache.hadoop.hbase.regionserver.RSRpcServices.scan(RSRpcServices.java:3637) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45819) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:392) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:140) at org.apache.hadoop.hbase.ipc.RpcExecutor$Handler.run(RpcExecutor.java:359) at org.apache.hadoop.hbase.ipc.RpcExecutor$Handler.run(RpcExecutor.java:339) "RpcServer.default.FPBQ.Fifo.handler=59,queue=9,port=16020" #146 daemon prio=5 os_prio=0 tid=0x00007fc11d867000 nid=0x7407 waiting for monitor entry [0x00007fc113fd1000] java.lang.Thread.State: BLOCKED (on object monitor) at org.apache.hadoop.fs.FSInputStream.read(FSInputStream.java:73) - waiting to lock <0x00007fc4c2769660> (a org.apache.hadoop.fs.s3a.S3AInputStream) at org.apache.hadoop.fs.FSDataInputStream.read(FSDataInputStream.java:97) at org.apache.hadoop.hbase.io.FileLink$FileLinkInputStream.read(FileLink.java:171) at org.apache.hadoop.fs.FSDataInputStream.read(FSDataInputStream.java:97) at org.apache.hadoop.hbase.io.util.BlockIOUtils.preadWithExtra(BlockIOUtils.java:235) at org.apache.hadoop.hbase.io.hfile.HFileBlock$FSReaderImpl.readAtOffset(HFileBlock.java:1564) at org.apache.hadoop.hbase.io.hfile.HFileBlock$FSReaderImpl.readBlockDataInternal(HFileBlock.java:1788) at org.apache.hadoop.hbase.io.hfile.HFileBlock$FSReaderImpl.readBlockData(HFileBlock.java:1597) at org.apache.hadoop.hbase.io.hfile.HFileReaderImpl.readBlock(HFileReaderImpl.java:1347) at org.apache.hadoop.hbase.io.hfile.HFileReaderImpl.readBlock(HFileReaderImpl.java:1268) at org.apache.hadoop.hbase.io.hfile.HFileReaderImpl$HFileScannerImpl.readNextDataBlock(HFileReaderImpl.java:762) at org.apache.hadoop.hbase.io.hfile.HFileReaderImpl$EncodedScanner.next(HFileReaderImpl.java:1540) at org.apache.hadoop.hbase.regionserver.StoreFileScanner.reseekAtOrAfter(StoreFileScanner.java:351) at org.apache.hadoop.hbase.regionserver.StoreFileScanner.reseek(StoreFileScanner.java:244) at org.apache.hadoop.hbase.regionserver.NonLazyKeyValueScanner.doRealSeek(NonLazyKeyValueScanner.java:55) at org.apache.hadoop.hbase.regionserver.KeyValueHeap.generalizedSeek(KeyValueHeap.java:322) at org.apache.hadoop.hbase.regionserver.KeyValueHeap.reseek(KeyValueHeap.java:265) at org.apache.hadoop.hbase.regionserver.StoreScanner.reseek(StoreScanner.java:1095) at org.apache.hadoop.hbase.regionserver.StoreScanner.seekAsDirection(StoreScanner.java:1084) at org.apache.hadoop.hbase.regionserver.StoreScanner.seekOrSkipToNextColumn(StoreScanner.java:821) at org.apache.hadoop.hbase.regionserver.StoreScanner.next(StoreScanner.java:684) at org.apache.hadoop.hbase.regionserver.KeyValueHeap.next(KeyValueHeap.java:155) at org.apache.hadoop.hbase.regionserver.HRegion$RegionScannerImpl.populateResult(HRegion.java:7399) at org.apache.hadoop.hbase.regionserver.HRegion$RegionScannerImpl.nextInternal(HRegion.java:7567) at org.apache.hadoop.hbase.regionserver.HRegion$RegionScannerImpl.nextRaw(HRegion.java:7331) at org.apache.hadoop.hbase.regionserver.HRegion$RegionScannerImpl.nextRaw(HRegion.java:7317) at org.apache.phoenix.coprocessor.PagedRegionScanner.next(PagedRegionScanner.java:57) at org.apache.phoenix.coprocessor.PagedRegionScanner.nextRaw(PagedRegionScanner.java:96) at org.apache.phoenix.iterate.RegionScannerFactory$1.nextRaw(RegionScannerFactory.java:211) at org.apache.phoenix.coprocessor.DelegateRegionScanner.nextRaw(DelegateRegionScanner.java:77) at org.apache.phoenix.coprocessor.DelegateRegionScanner.nextRaw(DelegateRegionScanner.java:77) at org.apache.phoenix.coprocessor.BaseScannerRegionObserver$RegionScannerHolder.nextRaw(BaseScannerRegionObserver.java:308) at org.apache.hadoop.hbase.regionserver.RSRpcServices.scan(RSRpcServices.java:3373) - locked <0x00007fc85f056e80> (a org.apache.phoenix.coprocessor.BaseScannerRegionObserver$RegionScannerHolder) at org.apache.hadoop.hbase.regionserver.RSRpcServices.scan(RSRpcServices.java:3637) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45819) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:392) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:140) at org.apache.hadoop.hbase.ipc.RpcExecutor$Handler.run(RpcExecutor.java:359) at org.apache.hadoop.hbase.ipc.RpcExecutor$Handler.run(RpcExecutor.java:339) "RpcServer.default.FPBQ.Fifo.handler=58,queue=8,port=16020" #145 daemon prio=5 os_prio=0 tid=0x00007fc11d865000 nid=0x7406 waiting for monitor entry [0x00007fc1140d2000] java.lang.Thread.State: BLOCKED (on object monitor) at org.apache.hadoop.fs.FSInputStream.read(FSInputStream.java:73) - waiting to lock <0x00007fc4c2769660> (a org.apache.hadoop.fs.s3a.S3AInputStream) at org.apache.hadoop.fs.FSDataInputStream.read(FSDataInputStream.java:97) at org.apache.hadoop.hbase.io.FileLink$FileLinkInputStream.read(FileLink.java:171) at org.apache.hadoop.fs.FSDataInputStream.read(FSDataInputStream.java:97) at org.apache.hadoop.hbase.io.util.BlockIOUtils.preadWithExtra(BlockIOUtils.java:235) at org.apache.hadoop.hbase.io.hfile.HFileBlock$FSReaderImpl.readAtOffset(HFileBlock.java:1564) at org.apache.hadoop.hbase.io.hfile.HFileBlock$FSReaderImpl.readBlockDataInternal(HFileBlock.java:1788) at org.apache.hadoop.hbase.io.hfile.HFileBlock$FSReaderImpl.readBlockData(HFileBlock.java:1597) at org.apache.hadoop.hbase.io.hfile.HFileReaderImpl.readBlock(HFileReaderImpl.java:1347) at org.apache.hadoop.hbase.io.hfile.HFileReaderImpl.readBlock(HFileReaderImpl.java:1268) at org.apache.hadoop.hbase.io.hfile.HFileReaderImpl$HFileScannerImpl.readNextDataBlock(HFileReaderImpl.java:762) at org.apache.hadoop.hbase.io.hfile.HFileReaderImpl$EncodedScanner.next(HFileReaderImpl.java:1540) at org.apache.hadoop.hbase.regionserver.StoreFileScanner.reseekAtOrAfter(StoreFileScanner.java:351) at org.apache.hadoop.hbase.regionserver.StoreFileScanner.reseek(StoreFileScanner.java:244) at org.apache.hadoop.hbase.regionserver.NonLazyKeyValueScanner.doRealSeek(NonLazyKeyValueScanner.java:55) at org.apache.hadoop.hbase.regionserver.KeyValueHeap.generalizedSeek(KeyValueHeap.java:322) at org.apache.hadoop.hbase.regionserver.KeyValueHeap.reseek(KeyValueHeap.java:265) at org.apache.hadoop.hbase.regionserver.StoreScanner.reseek(StoreScanner.java:1095) at org.apache.hadoop.hbase.regionserver.StoreScanner.seekAsDirection(StoreScanner.java:1084) at org.apache.hadoop.hbase.regionserver.StoreScanner.seekOrSkipToNextColumn(StoreScanner.java:821) at org.apache.hadoop.hbase.regionserver.StoreScanner.next(StoreScanner.java:684) at org.apache.hadoop.hbase.regionserver.KeyValueHeap.next(KeyValueHeap.java:155) at org.apache.hadoop.hbase.regionserver.HRegion$RegionScannerImpl.populateResult(HRegion.java:7399) at org.apache.hadoop.hbase.regionserver.HRegion$RegionScannerImpl.nextInternal(HRegion.java:7567) at org.apache.hadoop.hbase.regionserver.HRegion$RegionScannerImpl.nextRaw(HRegion.java:7331) at org.apache.hadoop.hbase.regionserver.HRegion$RegionScannerImpl.nextRaw(HRegion.java:7317) at org.apache.phoenix.coprocessor.PagedRegionScanner.next(PagedRegionScanner.java:57) at org.apache.phoenix.coprocessor.PagedRegionScanner.nextRaw(PagedRegionScanner.java:96) at org.apache.phoenix.iterate.RegionScannerFactory$1.nextRaw(RegionScannerFactory.java:211) at org.apache.phoenix.coprocessor.DelegateRegionScanner.nextRaw(DelegateRegionScanner.java:77) at org.apache.phoenix.coprocessor.DelegateRegionScanner.nextRaw(DelegateRegionScanner.java:77) at org.apache.phoenix.coprocessor.BaseScannerRegionObserver$RegionScannerHolder.nextRaw(BaseScannerRegionObserver.java:308) at org.apache.hadoop.hbase.regionserver.RSRpcServices.scan(RSRpcServices.java:3373) - locked <0x00007fc85f0469d0> (a org.apache.phoenix.coprocessor.BaseScannerRegionObserver$RegionScannerHolder) at org.apache.hadoop.hbase.regionserver.RSRpcServices.scan(RSRpcServices.java:3637) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45819) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:392) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:140) at org.apache.hadoop.hbase.ipc.RpcExecutor$Handler.run(RpcExecutor.java:359) at org.apache.hadoop.hbase.ipc.RpcExecutor$Handler.run(RpcExecutor.java:339) "RpcServer.default.FPBQ.Fifo.handler=57,queue=7,port=16020" #144 daemon prio=5 os_prio=0 tid=0x00007fc11d863000 nid=0x7405 waiting for monitor entry [0x00007fc1141d3000] java.lang.Thread.State: BLOCKED (on object monitor) at org.apache.hadoop.fs.FSInputStream.read(FSInputStream.java:73) - waiting to lock <0x00007fc4c2769660> (a org.apache.hadoop.fs.s3a.S3AInputStream) at org.apache.hadoop.fs.FSDataInputStream.read(FSDataInputStream.java:97) at org.apache.hadoop.hbase.io.FileLink$FileLinkInputStream.read(FileLink.java:171) at org.apache.hadoop.fs.FSDataInputStream.read(FSDataInputStream.java:97) at org.apache.hadoop.hbase.io.util.BlockIOUtils.preadWithExtra(BlockIOUtils.java:235) at org.apache.hadoop.hbase.io.hfile.HFileBlock$FSReaderImpl.readAtOffset(HFileBlock.java:1564) at org.apache.hadoop.hbase.io.hfile.HFileBlock$FSReaderImpl.readBlockDataInternal(HFileBlock.java:1788) at org.apache.hadoop.hbase.io.hfile.HFileBlock$FSReaderImpl.readBlockData(HFileBlock.java:1597) at org.apache.hadoop.hbase.io.hfile.HFileReaderImpl.readBlock(HFileReaderImpl.java:1347) at org.apache.hadoop.hbase.io.hfile.HFileReaderImpl.readBlock(HFileReaderImpl.java:1268) at org.apache.hadoop.hbase.io.hfile.HFileReaderImpl$HFileScannerImpl.readNextDataBlock(HFileReaderImpl.java:762) at org.apache.hadoop.hbase.io.hfile.HFileReaderImpl$EncodedScanner.next(HFileReaderImpl.java:1540) at org.apache.hadoop.hbase.regionserver.StoreFileScanner.reseekAtOrAfter(StoreFileScanner.java:351) at org.apache.hadoop.hbase.regionserver.StoreFileScanner.reseek(StoreFileScanner.java:244) at org.apache.hadoop.hbase.regionserver.NonLazyKeyValueScanner.doRealSeek(NonLazyKeyValueScanner.java:55) at org.apache.hadoop.hbase.regionserver.KeyValueHeap.generalizedSeek(KeyValueHeap.java:322) at org.apache.hadoop.hbase.regionserver.KeyValueHeap.reseek(KeyValueHeap.java:265) at org.apache.hadoop.hbase.regionserver.StoreScanner.reseek(StoreScanner.java:1095) at org.apache.hadoop.hbase.regionserver.StoreScanner.seekAsDirection(StoreScanner.java:1084) at org.apache.hadoop.hbase.regionserver.StoreScanner.seekOrSkipToNextColumn(StoreScanner.java:821) at org.apache.hadoop.hbase.regionserver.StoreScanner.next(StoreScanner.java:684) at org.apache.hadoop.hbase.regionserver.KeyValueHeap.next(KeyValueHeap.java:155) at org.apache.hadoop.hbase.regionserver.HRegion$RegionScannerImpl.populateResult(HRegion.java:7399) at org.apache.hadoop.hbase.regionserver.HRegion$RegionScannerImpl.nextInternal(HRegion.java:7567) at org.apache.hadoop.hbase.regionserver.HRegion$RegionScannerImpl.nextRaw(HRegion.java:7331) at org.apache.hadoop.hbase.regionserver.HRegion$RegionScannerImpl.nextRaw(HRegion.java:7317) at org.apache.phoenix.coprocessor.PagedRegionScanner.next(PagedRegionScanner.java:57) at org.apache.phoenix.coprocessor.PagedRegionScanner.nextRaw(PagedRegionScanner.java:96) at org.apache.phoenix.iterate.RegionScannerFactory$1.nextRaw(RegionScannerFactory.java:211) at org.apache.phoenix.coprocessor.DelegateRegionScanner.nextRaw(DelegateRegionScanner.java:77) at org.apache.phoenix.coprocessor.DelegateRegionScanner.nextRaw(DelegateRegionScanner.java:77) at org.apache.phoenix.coprocessor.BaseScannerRegionObserver$RegionScannerHolder.nextRaw(BaseScannerRegionObserver.java:308) at org.apache.hadoop.hbase.regionserver.RSRpcServices.scan(RSRpcServices.java:3373) - locked <0x00007fc85f017ce8> (a org.apache.phoenix.coprocessor.BaseScannerRegionObserver$RegionScannerHolder) at org.apache.hadoop.hbase.regionserver.RSRpcServices.scan(RSRpcServices.java:3637) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45819) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:392) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:140) at org.apache.hadoop.hbase.ipc.RpcExecutor$Handler.run(RpcExecutor.java:359) at org.apache.hadoop.hbase.ipc.RpcExecutor$Handler.run(RpcExecutor.java:339) "RpcServer.default.FPBQ.Fifo.handler=56,queue=6,port=16020" #143 daemon prio=5 os_prio=0 tid=0x00007fc11d860800 nid=0x7404 waiting for monitor entry [0x00007fc1142d4000] java.lang.Thread.State: BLOCKED (on object monitor) at org.apache.hadoop.fs.FSInputStream.read(FSInputStream.java:73) - waiting to lock <0x00007fc4c2769660> (a org.apache.hadoop.fs.s3a.S3AInputStream) at org.apache.hadoop.fs.FSDataInputStream.read(FSDataInputStream.java:97) at org.apache.hadoop.hbase.io.FileLink$FileLinkInputStream.read(FileLink.java:171) at org.apache.hadoop.fs.FSDataInputStream.read(FSDataInputStream.java:97) at org.apache.hadoop.hbase.io.util.BlockIOUtils.preadWithExtra(BlockIOUtils.java:235) at org.apache.hadoop.hbase.io.hfile.HFileBlock$FSReaderImpl.readAtOffset(HFileBlock.java:1564) at org.apache.hadoop.hbase.io.hfile.HFileBlock$FSReaderImpl.readBlockDataInternal(HFileBlock.java:1788) at org.apache.hadoop.hbase.io.hfile.HFileBlock$FSReaderImpl.readBlockData(HFileBlock.java:1597) at org.apache.hadoop.hbase.io.hfile.HFileReaderImpl.readBlock(HFileReaderImpl.java:1347) at org.apache.hadoop.hbase.io.hfile.HFileReaderImpl.readBlock(HFileReaderImpl.java:1268) at org.apache.hadoop.hbase.io.hfile.HFileReaderImpl$HFileScannerImpl.readNextDataBlock(HFileReaderImpl.java:762) at org.apache.hadoop.hbase.io.hfile.HFileReaderImpl$EncodedScanner.next(HFileReaderImpl.java:1540) at org.apache.hadoop.hbase.regionserver.StoreFileScanner.reseekAtOrAfter(StoreFileScanner.java:351) at org.apache.hadoop.hbase.regionserver.StoreFileScanner.reseek(StoreFileScanner.java:244) at org.apache.hadoop.hbase.regionserver.NonLazyKeyValueScanner.doRealSeek(NonLazyKeyValueScanner.java:55) at org.apache.hadoop.hbase.regionserver.KeyValueHeap.generalizedSeek(KeyValueHeap.java:322) at org.apache.hadoop.hbase.regionserver.KeyValueHeap.reseek(KeyValueHeap.java:265) at org.apache.hadoop.hbase.regionserver.StoreScanner.reseek(StoreScanner.java:1095) at org.apache.hadoop.hbase.regionserver.StoreScanner.seekAsDirection(StoreScanner.java:1084) at org.apache.hadoop.hbase.regionserver.StoreScanner.seekOrSkipToNextColumn(StoreScanner.java:821) at org.apache.hadoop.hbase.regionserver.StoreScanner.next(StoreScanner.java:684) at org.apache.hadoop.hbase.regionserver.KeyValueHeap.next(KeyValueHeap.java:155) at org.apache.hadoop.hbase.regionserver.HRegion$RegionScannerImpl.populateResult(HRegion.java:7399) at org.apache.hadoop.hbase.regionserver.HRegion$RegionScannerImpl.nextInternal(HRegion.java:7567) at org.apache.hadoop.hbase.regionserver.HRegion$RegionScannerImpl.nextRaw(HRegion.java:7331) at org.apache.hadoop.hbase.regionserver.HRegion$RegionScannerImpl.nextRaw(HRegion.java:7317) at org.apache.phoenix.coprocessor.PagedRegionScanner.next(PagedRegionScanner.java:57) at org.apache.phoenix.coprocessor.PagedRegionScanner.nextRaw(PagedRegionScanner.java:96) at org.apache.phoenix.iterate.RegionScannerFactory$1.nextRaw(RegionScannerFactory.java:211) at org.apache.phoenix.coprocessor.DelegateRegionScanner.nextRaw(DelegateRegionScanner.java:77) at org.apache.phoenix.coprocessor.DelegateRegionScanner.nextRaw(DelegateRegionScanner.java:77) at org.apache.phoenix.coprocessor.BaseScannerRegionObserver$RegionScannerHolder.nextRaw(BaseScannerRegionObserver.java:308) at org.apache.hadoop.hbase.regionserver.RSRpcServices.scan(RSRpcServices.java:3373) - locked <0x00007fc85f074288> (a org.apache.phoenix.coprocessor.BaseScannerRegionObserver$RegionScannerHolder) at org.apache.hadoop.hbase.regionserver.RSRpcServices.scan(RSRpcServices.java:3637) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45819) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:392) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:140) at org.apache.hadoop.hbase.ipc.RpcExecutor$Handler.run(RpcExecutor.java:359) at org.apache.hadoop.hbase.ipc.RpcExecutor$Handler.run(RpcExecutor.java:339) "RpcServer.default.FPBQ.Fifo.handler=55,queue=5,port=16020" #142 daemon prio=5 os_prio=0 tid=0x00007fc11d85e800 nid=0x7403 waiting on condition [0x00007fc1143d6000] java.lang.Thread.State: WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007fc1c9ff03e0> (a java.util.concurrent.Semaphore$NonfairSync) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer.parkAndCheckInterrupt(AbstractQueuedSynchronizer.java:836) at java.util.concurrent.locks.AbstractQueuedSynchronizer.doAcquireSharedInterruptibly(AbstractQueuedSynchronizer.java:997) at java.util.concurrent.locks.AbstractQueuedSynchronizer.acquireSharedInterruptibly(AbstractQueuedSynchronizer.java:1304) at java.util.concurrent.Semaphore.acquire(Semaphore.java:312) at org.apache.hadoop.hbase.ipc.FastPathBalancedQueueRpcExecutor$FastPathHandler.getCallRunner(FastPathBalancedQueueRpcExecutor.java:109) at org.apache.hadoop.hbase.ipc.RpcExecutor$Handler.run(RpcExecutor.java:339) "RpcServer.default.FPBQ.Fifo.handler=54,queue=4,port=16020" #141 daemon prio=5 os_prio=0 tid=0x00007fc11d85c800 nid=0x7402 waiting for monitor entry [0x00007fc1144d6000] java.lang.Thread.State: BLOCKED (on object monitor) at org.apache.hadoop.fs.FSInputStream.read(FSInputStream.java:73) - waiting to lock <0x00007fc4c2769660> (a org.apache.hadoop.fs.s3a.S3AInputStream) at org.apache.hadoop.fs.FSDataInputStream.read(FSDataInputStream.java:97) at org.apache.hadoop.hbase.io.FileLink$FileLinkInputStream.read(FileLink.java:171) at org.apache.hadoop.fs.FSDataInputStream.read(FSDataInputStream.java:97) at org.apache.hadoop.hbase.io.util.BlockIOUtils.preadWithExtra(BlockIOUtils.java:235) at org.apache.hadoop.hbase.io.hfile.HFileBlock$FSReaderImpl.readAtOffset(HFileBlock.java:1564) at org.apache.hadoop.hbase.io.hfile.HFileBlock$FSReaderImpl.readBlockDataInternal(HFileBlock.java:1788) at org.apache.hadoop.hbase.io.hfile.HFileBlock$FSReaderImpl.readBlockData(HFileBlock.java:1597) at org.apache.hadoop.hbase.io.hfile.HFileReaderImpl.readBlock(HFileReaderImpl.java:1347) at org.apache.hadoop.hbase.io.hfile.HFileReaderImpl.readBlock(HFileReaderImpl.java:1268) at org.apache.hadoop.hbase.io.hfile.HFileReaderImpl$HFileScannerImpl.readNextDataBlock(HFileReaderImpl.java:762) at org.apache.hadoop.hbase.io.hfile.HFileReaderImpl$EncodedScanner.next(HFileReaderImpl.java:1540) at org.apache.hadoop.hbase.regionserver.StoreFileScanner.reseekAtOrAfter(StoreFileScanner.java:351) at org.apache.hadoop.hbase.regionserver.StoreFileScanner.reseek(StoreFileScanner.java:244) at org.apache.hadoop.hbase.regionserver.NonLazyKeyValueScanner.doRealSeek(NonLazyKeyValueScanner.java:55) at org.apache.hadoop.hbase.regionserver.KeyValueHeap.generalizedSeek(KeyValueHeap.java:322) at org.apache.hadoop.hbase.regionserver.KeyValueHeap.reseek(KeyValueHeap.java:265) at org.apache.hadoop.hbase.regionserver.StoreScanner.reseek(StoreScanner.java:1095) at org.apache.hadoop.hbase.regionserver.StoreScanner.seekAsDirection(StoreScanner.java:1084) at org.apache.hadoop.hbase.regionserver.StoreScanner.seekOrSkipToNextColumn(StoreScanner.java:821) at org.apache.hadoop.hbase.regionserver.StoreScanner.next(StoreScanner.java:684) at org.apache.hadoop.hbase.regionserver.KeyValueHeap.next(KeyValueHeap.java:155) at org.apache.hadoop.hbase.regionserver.HRegion$RegionScannerImpl.populateResult(HRegion.java:7399) at org.apache.hadoop.hbase.regionserver.HRegion$RegionScannerImpl.nextInternal(HRegion.java:7567) at org.apache.hadoop.hbase.regionserver.HRegion$RegionScannerImpl.nextRaw(HRegion.java:7331) at org.apache.hadoop.hbase.regionserver.HRegion$RegionScannerImpl.nextRaw(HRegion.java:7317) at org.apache.phoenix.coprocessor.PagedRegionScanner.next(PagedRegionScanner.java:57) at org.apache.phoenix.coprocessor.PagedRegionScanner.nextRaw(PagedRegionScanner.java:96) at org.apache.phoenix.iterate.RegionScannerFactory$1.nextRaw(RegionScannerFactory.java:211) at org.apache.phoenix.coprocessor.DelegateRegionScanner.nextRaw(DelegateRegionScanner.java:77) at org.apache.phoenix.coprocessor.DelegateRegionScanner.nextRaw(DelegateRegionScanner.java:77) at org.apache.phoenix.coprocessor.BaseScannerRegionObserver$RegionScannerHolder.nextRaw(BaseScannerRegionObserver.java:308) at org.apache.hadoop.hbase.regionserver.RSRpcServices.scan(RSRpcServices.java:3373) - locked <0x00007fc85f037340> (a org.apache.phoenix.coprocessor.BaseScannerRegionObserver$RegionScannerHolder) at org.apache.hadoop.hbase.regionserver.RSRpcServices.scan(RSRpcServices.java:3637) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45819) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:392) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:140) at org.apache.hadoop.hbase.ipc.RpcExecutor$Handler.run(RpcExecutor.java:359) at org.apache.hadoop.hbase.ipc.RpcExecutor$Handler.run(RpcExecutor.java:339) "RpcServer.default.FPBQ.Fifo.handler=53,queue=3,port=16020" #140 daemon prio=5 os_prio=0 tid=0x00007fc11d85a800 nid=0x7401 waiting for monitor entry [0x00007fc1145d7000] java.lang.Thread.State: BLOCKED (on object monitor) at org.apache.hadoop.fs.FSInputStream.read(FSInputStream.java:73) - waiting to lock <0x00007fc4c2769660> (a org.apache.hadoop.fs.s3a.S3AInputStream) at org.apache.hadoop.fs.FSDataInputStream.read(FSDataInputStream.java:97) at org.apache.hadoop.hbase.io.FileLink$FileLinkInputStream.read(FileLink.java:171) at org.apache.hadoop.fs.FSDataInputStream.read(FSDataInputStream.java:97) at org.apache.hadoop.hbase.io.util.BlockIOUtils.preadWithExtra(BlockIOUtils.java:235) at org.apache.hadoop.hbase.io.hfile.HFileBlock$FSReaderImpl.readAtOffset(HFileBlock.java:1564) at org.apache.hadoop.hbase.io.hfile.HFileBlock$FSReaderImpl.readBlockDataInternal(HFileBlock.java:1788) at org.apache.hadoop.hbase.io.hfile.HFileBlock$FSReaderImpl.readBlockData(HFileBlock.java:1597) at org.apache.hadoop.hbase.io.hfile.HFileReaderImpl.readBlock(HFileReaderImpl.java:1347) at org.apache.hadoop.hbase.io.hfile.HFileReaderImpl.readBlock(HFileReaderImpl.java:1268) at org.apache.hadoop.hbase.io.hfile.HFileReaderImpl$HFileScannerImpl.readNextDataBlock(HFileReaderImpl.java:762) at org.apache.hadoop.hbase.io.hfile.HFileReaderImpl$EncodedScanner.next(HFileReaderImpl.java:1540) at org.apache.hadoop.hbase.regionserver.StoreFileScanner.reseekAtOrAfter(StoreFileScanner.java:351) at org.apache.hadoop.hbase.regionserver.StoreFileScanner.reseek(StoreFileScanner.java:244) at org.apache.hadoop.hbase.regionserver.NonLazyKeyValueScanner.doRealSeek(NonLazyKeyValueScanner.java:55) at org.apache.hadoop.hbase.regionserver.KeyValueHeap.generalizedSeek(KeyValueHeap.java:322) at org.apache.hadoop.hbase.regionserver.KeyValueHeap.reseek(KeyValueHeap.java:265) at org.apache.hadoop.hbase.regionserver.StoreScanner.reseek(StoreScanner.java:1095) at org.apache.hadoop.hbase.regionserver.StoreScanner.seekAsDirection(StoreScanner.java:1084) at org.apache.hadoop.hbase.regionserver.StoreScanner.seekOrSkipToNextColumn(StoreScanner.java:821) at org.apache.hadoop.hbase.regionserver.StoreScanner.next(StoreScanner.java:684) at org.apache.hadoop.hbase.regionserver.KeyValueHeap.next(KeyValueHeap.java:155) at org.apache.hadoop.hbase.regionserver.HRegion$RegionScannerImpl.populateResult(HRegion.java:7399) at org.apache.hadoop.hbase.regionserver.HRegion$RegionScannerImpl.nextInternal(HRegion.java:7567) at org.apache.hadoop.hbase.regionserver.HRegion$RegionScannerImpl.nextRaw(HRegion.java:7331) at org.apache.hadoop.hbase.regionserver.HRegion$RegionScannerImpl.nextRaw(HRegion.java:7317) at org.apache.phoenix.coprocessor.PagedRegionScanner.next(PagedRegionScanner.java:57) at org.apache.phoenix.coprocessor.PagedRegionScanner.nextRaw(PagedRegionScanner.java:96) at org.apache.phoenix.iterate.RegionScannerFactory$1.nextRaw(RegionScannerFactory.java:211) at org.apache.phoenix.coprocessor.DelegateRegionScanner.nextRaw(DelegateRegionScanner.java:77) at org.apache.phoenix.coprocessor.DelegateRegionScanner.nextRaw(DelegateRegionScanner.java:77) at org.apache.phoenix.coprocessor.BaseScannerRegionObserver$RegionScannerHolder.nextRaw(BaseScannerRegionObserver.java:308) at org.apache.hadoop.hbase.regionserver.RSRpcServices.scan(RSRpcServices.java:3373) - locked <0x00007fc85f059048> (a org.apache.phoenix.coprocessor.BaseScannerRegionObserver$RegionScannerHolder) at org.apache.hadoop.hbase.regionserver.RSRpcServices.scan(RSRpcServices.java:3637) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45819) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:392) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:140) at org.apache.hadoop.hbase.ipc.RpcExecutor$Handler.run(RpcExecutor.java:359) at org.apache.hadoop.hbase.ipc.RpcExecutor$Handler.run(RpcExecutor.java:339) "RpcServer.default.FPBQ.Fifo.handler=52,queue=2,port=16020" #139 daemon prio=5 os_prio=0 tid=0x00007fc11d858000 nid=0x7400 waiting for monitor entry [0x00007fc1146d8000] java.lang.Thread.State: BLOCKED (on object monitor) at org.apache.hadoop.fs.FSInputStream.read(FSInputStream.java:73) - waiting to lock <0x00007fc4c2769660> (a org.apache.hadoop.fs.s3a.S3AInputStream) at org.apache.hadoop.fs.FSDataInputStream.read(FSDataInputStream.java:97) at org.apache.hadoop.hbase.io.FileLink$FileLinkInputStream.read(FileLink.java:171) at org.apache.hadoop.fs.FSDataInputStream.read(FSDataInputStream.java:97) at org.apache.hadoop.hbase.io.util.BlockIOUtils.preadWithExtra(BlockIOUtils.java:235) at org.apache.hadoop.hbase.io.hfile.HFileBlock$FSReaderImpl.readAtOffset(HFileBlock.java:1564) at org.apache.hadoop.hbase.io.hfile.HFileBlock$FSReaderImpl.readBlockDataInternal(HFileBlock.java:1788) at org.apache.hadoop.hbase.io.hfile.HFileBlock$FSReaderImpl.readBlockData(HFileBlock.java:1597) at org.apache.hadoop.hbase.io.hfile.HFileReaderImpl.readBlock(HFileReaderImpl.java:1347) at org.apache.hadoop.hbase.io.hfile.HFileReaderImpl.readBlock(HFileReaderImpl.java:1268) at org.apache.hadoop.hbase.io.hfile.HFileReaderImpl$HFileScannerImpl.readNextDataBlock(HFileReaderImpl.java:762) at org.apache.hadoop.hbase.io.hfile.HFileReaderImpl$EncodedScanner.next(HFileReaderImpl.java:1540) at org.apache.hadoop.hbase.regionserver.StoreFileScanner.next(StoreFileScanner.java:196) at org.apache.hadoop.hbase.regionserver.KeyValueHeap.next(KeyValueHeap.java:122) at org.apache.hadoop.hbase.regionserver.StoreScanner.trySkipToNextRow(StoreScanner.java:884) at org.apache.hadoop.hbase.regionserver.StoreScanner.seekOrSkipToNextRow(StoreScanner.java:812) at org.apache.hadoop.hbase.regionserver.StoreScanner.next(StoreScanner.java:720) at org.apache.hadoop.hbase.regionserver.KeyValueHeap.next(KeyValueHeap.java:155) at org.apache.hadoop.hbase.regionserver.HRegion$RegionScannerImpl.populateResult(HRegion.java:7399) at org.apache.hadoop.hbase.regionserver.HRegion$RegionScannerImpl.nextInternal(HRegion.java:7567) at org.apache.hadoop.hbase.regionserver.HRegion$RegionScannerImpl.nextRaw(HRegion.java:7331) at org.apache.hadoop.hbase.regionserver.HRegion$RegionScannerImpl.nextRaw(HRegion.java:7317) at org.apache.phoenix.coprocessor.PagedRegionScanner.next(PagedRegionScanner.java:57) at org.apache.phoenix.coprocessor.PagedRegionScanner.nextRaw(PagedRegionScanner.java:96) at org.apache.phoenix.iterate.RegionScannerFactory$1.nextRaw(RegionScannerFactory.java:211) at org.apache.phoenix.coprocessor.DelegateRegionScanner.nextRaw(DelegateRegionScanner.java:77) at org.apache.phoenix.coprocessor.DelegateRegionScanner.nextRaw(DelegateRegionScanner.java:77) at org.apache.phoenix.coprocessor.BaseScannerRegionObserver$RegionScannerHolder.nextRaw(BaseScannerRegionObserver.java:308) at org.apache.hadoop.hbase.regionserver.RSRpcServices.scan(RSRpcServices.java:3373) - locked <0x00007fc85f05e9a8> (a org.apache.phoenix.coprocessor.BaseScannerRegionObserver$RegionScannerHolder) at org.apache.hadoop.hbase.regionserver.RSRpcServices.scan(RSRpcServices.java:3637) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45819) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:392) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:140) at org.apache.hadoop.hbase.ipc.RpcExecutor$Handler.run(RpcExecutor.java:359) at org.apache.hadoop.hbase.ipc.RpcExecutor$Handler.run(RpcExecutor.java:339) "RpcServer.default.FPBQ.Fifo.handler=51,queue=1,port=16020" #138 daemon prio=5 os_prio=0 tid=0x00007fc11d856000 nid=0x73ff waiting for monitor entry [0x00007fc1147d9000] java.lang.Thread.State: BLOCKED (on object monitor) at org.apache.hadoop.fs.FSInputStream.read(FSInputStream.java:73) - waiting to lock <0x00007fc4c2769660> (a org.apache.hadoop.fs.s3a.S3AInputStream) at org.apache.hadoop.fs.FSDataInputStream.read(FSDataInputStream.java:97) at org.apache.hadoop.hbase.io.FileLink$FileLinkInputStream.read(FileLink.java:171) at org.apache.hadoop.fs.FSDataInputStream.read(FSDataInputStream.java:97) at org.apache.hadoop.hbase.io.util.BlockIOUtils.preadWithExtra(BlockIOUtils.java:235) at org.apache.hadoop.hbase.io.hfile.HFileBlock$FSReaderImpl.readAtOffset(HFileBlock.java:1564) at org.apache.hadoop.hbase.io.hfile.HFileBlock$FSReaderImpl.readBlockDataInternal(HFileBlock.java:1788) at org.apache.hadoop.hbase.io.hfile.HFileBlock$FSReaderImpl.readBlockData(HFileBlock.java:1597) at org.apache.hadoop.hbase.io.hfile.HFileReaderImpl.readBlock(HFileReaderImpl.java:1347) at org.apache.hadoop.hbase.io.hfile.HFileReaderImpl.readBlock(HFileReaderImpl.java:1268) at org.apache.hadoop.hbase.io.hfile.HFileReaderImpl$HFileScannerImpl.readNextDataBlock(HFileReaderImpl.java:762) at org.apache.hadoop.hbase.io.hfile.HFileReaderImpl$EncodedScanner.next(HFileReaderImpl.java:1540) at org.apache.hadoop.hbase.regionserver.StoreFileScanner.reseekAtOrAfter(StoreFileScanner.java:351) at org.apache.hadoop.hbase.regionserver.StoreFileScanner.reseek(StoreFileScanner.java:244) at org.apache.hadoop.hbase.regionserver.NonLazyKeyValueScanner.doRealSeek(NonLazyKeyValueScanner.java:55) at org.apache.hadoop.hbase.regionserver.KeyValueHeap.generalizedSeek(KeyValueHeap.java:322) at org.apache.hadoop.hbase.regionserver.KeyValueHeap.reseek(KeyValueHeap.java:265) at org.apache.hadoop.hbase.regionserver.StoreScanner.reseek(StoreScanner.java:1095) at org.apache.hadoop.hbase.regionserver.StoreScanner.seekAsDirection(StoreScanner.java:1084) at org.apache.hadoop.hbase.regionserver.StoreScanner.seekOrSkipToNextColumn(StoreScanner.java:821) at org.apache.hadoop.hbase.regionserver.StoreScanner.next(StoreScanner.java:684) at org.apache.hadoop.hbase.regionserver.KeyValueHeap.next(KeyValueHeap.java:155) at org.apache.hadoop.hbase.regionserver.HRegion$RegionScannerImpl.populateResult(HRegion.java:7399) at org.apache.hadoop.hbase.regionserver.HRegion$RegionScannerImpl.nextInternal(HRegion.java:7567) at org.apache.hadoop.hbase.regionserver.HRegion$RegionScannerImpl.nextRaw(HRegion.java:7331) at org.apache.hadoop.hbase.regionserver.HRegion$RegionScannerImpl.nextRaw(HRegion.java:7317) at org.apache.phoenix.coprocessor.PagedRegionScanner.next(PagedRegionScanner.java:57) at org.apache.phoenix.coprocessor.PagedRegionScanner.nextRaw(PagedRegionScanner.java:96) at org.apache.phoenix.iterate.RegionScannerFactory$1.nextRaw(RegionScannerFactory.java:211) at org.apache.phoenix.coprocessor.DelegateRegionScanner.nextRaw(DelegateRegionScanner.java:77) at org.apache.phoenix.coprocessor.DelegateRegionScanner.nextRaw(DelegateRegionScanner.java:77) at org.apache.phoenix.coprocessor.BaseScannerRegionObserver$RegionScannerHolder.nextRaw(BaseScannerRegionObserver.java:308) at org.apache.hadoop.hbase.regionserver.RSRpcServices.scan(RSRpcServices.java:3373) - locked <0x00007fc85f0763e8> (a org.apache.phoenix.coprocessor.BaseScannerRegionObserver$RegionScannerHolder) at org.apache.hadoop.hbase.regionserver.RSRpcServices.scan(RSRpcServices.java:3637) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45819) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:392) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:140) at org.apache.hadoop.hbase.ipc.RpcExecutor$Handler.run(RpcExecutor.java:359) at org.apache.hadoop.hbase.ipc.RpcExecutor$Handler.run(RpcExecutor.java:339) "RpcServer.default.FPBQ.Fifo.handler=50,queue=0,port=16020" #137 daemon prio=5 os_prio=0 tid=0x00007fc11d854000 nid=0x73fe waiting on condition [0x00007fc1148db000] java.lang.Thread.State: WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007fc1c9ff05f0> (a java.util.concurrent.Semaphore$NonfairSync) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer.parkAndCheckInterrupt(AbstractQueuedSynchronizer.java:836) at java.util.concurrent.locks.AbstractQueuedSynchronizer.doAcquireSharedInterruptibly(AbstractQueuedSynchronizer.java:997) at java.util.concurrent.locks.AbstractQueuedSynchronizer.acquireSharedInterruptibly(AbstractQueuedSynchronizer.java:1304) at java.util.concurrent.Semaphore.acquire(Semaphore.java:312) at org.apache.hadoop.hbase.ipc.FastPathBalancedQueueRpcExecutor$FastPathHandler.getCallRunner(FastPathBalancedQueueRpcExecutor.java:109) at org.apache.hadoop.hbase.ipc.RpcExecutor$Handler.run(RpcExecutor.java:339) "RpcServer.default.FPBQ.Fifo.handler=49,queue=9,port=16020" #136 daemon prio=5 os_prio=0 tid=0x00007fc11d852800 nid=0x73fd waiting for monitor entry [0x00007fc1149db000] java.lang.Thread.State: BLOCKED (on object monitor) at org.apache.hadoop.fs.FSInputStream.read(FSInputStream.java:73) - waiting to lock <0x00007fc4c2769660> (a org.apache.hadoop.fs.s3a.S3AInputStream) at org.apache.hadoop.fs.FSDataInputStream.read(FSDataInputStream.java:97) at org.apache.hadoop.hbase.io.FileLink$FileLinkInputStream.read(FileLink.java:171) at org.apache.hadoop.fs.FSDataInputStream.read(FSDataInputStream.java:97) at org.apache.hadoop.hbase.io.util.BlockIOUtils.preadWithExtra(BlockIOUtils.java:235) at org.apache.hadoop.hbase.io.hfile.HFileBlock$FSReaderImpl.readAtOffset(HFileBlock.java:1564) at org.apache.hadoop.hbase.io.hfile.HFileBlock$FSReaderImpl.readBlockDataInternal(HFileBlock.java:1788) at org.apache.hadoop.hbase.io.hfile.HFileBlock$FSReaderImpl.readBlockData(HFileBlock.java:1597) at org.apache.hadoop.hbase.io.hfile.HFileReaderImpl.readBlock(HFileReaderImpl.java:1347) at org.apache.hadoop.hbase.io.hfile.HFileReaderImpl.readBlock(HFileReaderImpl.java:1268) at org.apache.hadoop.hbase.io.hfile.HFileReaderImpl$HFileScannerImpl.readNextDataBlock(HFileReaderImpl.java:762) at org.apache.hadoop.hbase.io.hfile.HFileReaderImpl$EncodedScanner.next(HFileReaderImpl.java:1540) at org.apache.hadoop.hbase.regionserver.StoreFileScanner.reseekAtOrAfter(StoreFileScanner.java:351) at org.apache.hadoop.hbase.regionserver.StoreFileScanner.reseek(StoreFileScanner.java:244) at org.apache.hadoop.hbase.regionserver.NonLazyKeyValueScanner.doRealSeek(NonLazyKeyValueScanner.java:55) at org.apache.hadoop.hbase.regionserver.KeyValueHeap.generalizedSeek(KeyValueHeap.java:322) at org.apache.hadoop.hbase.regionserver.KeyValueHeap.reseek(KeyValueHeap.java:265) at org.apache.hadoop.hbase.regionserver.StoreScanner.reseek(StoreScanner.java:1095) at org.apache.hadoop.hbase.regionserver.StoreScanner.seekAsDirection(StoreScanner.java:1084) at org.apache.hadoop.hbase.regionserver.StoreScanner.seekOrSkipToNextColumn(StoreScanner.java:821) at org.apache.hadoop.hbase.regionserver.StoreScanner.next(StoreScanner.java:684) at org.apache.hadoop.hbase.regionserver.KeyValueHeap.next(KeyValueHeap.java:155) at org.apache.hadoop.hbase.regionserver.HRegion$RegionScannerImpl.populateResult(HRegion.java:7399) at org.apache.hadoop.hbase.regionserver.HRegion$RegionScannerImpl.nextInternal(HRegion.java:7567) at org.apache.hadoop.hbase.regionserver.HRegion$RegionScannerImpl.nextRaw(HRegion.java:7331) at org.apache.hadoop.hbase.regionserver.HRegion$RegionScannerImpl.nextRaw(HRegion.java:7317) at org.apache.phoenix.coprocessor.PagedRegionScanner.next(PagedRegionScanner.java:57) at org.apache.phoenix.coprocessor.PagedRegionScanner.nextRaw(PagedRegionScanner.java:96) at org.apache.phoenix.iterate.RegionScannerFactory$1.nextRaw(RegionScannerFactory.java:211) at org.apache.phoenix.coprocessor.DelegateRegionScanner.nextRaw(DelegateRegionScanner.java:77) at org.apache.phoenix.coprocessor.DelegateRegionScanner.nextRaw(DelegateRegionScanner.java:77) at org.apache.phoenix.coprocessor.BaseScannerRegionObserver$RegionScannerHolder.nextRaw(BaseScannerRegionObserver.java:308) at org.apache.hadoop.hbase.regionserver.RSRpcServices.scan(RSRpcServices.java:3373) - locked <0x00007fc85f060b08> (a org.apache.phoenix.coprocessor.BaseScannerRegionObserver$RegionScannerHolder) at org.apache.hadoop.hbase.regionserver.RSRpcServices.scan(RSRpcServices.java:3637) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45819) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:392) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:140) at org.apache.hadoop.hbase.ipc.RpcExecutor$Handler.run(RpcExecutor.java:359) at org.apache.hadoop.hbase.ipc.RpcExecutor$Handler.run(RpcExecutor.java:339) "RpcServer.default.FPBQ.Fifo.handler=48,queue=8,port=16020" #135 daemon prio=5 os_prio=0 tid=0x00007fc11d850800 nid=0x73fc waiting for monitor entry [0x00007fc114adc000] java.lang.Thread.State: BLOCKED (on object monitor) at org.apache.hadoop.fs.FSInputStream.read(FSInputStream.java:73) - waiting to lock <0x00007fc4c2769660> (a org.apache.hadoop.fs.s3a.S3AInputStream) at org.apache.hadoop.fs.FSDataInputStream.read(FSDataInputStream.java:97) at org.apache.hadoop.hbase.io.FileLink$FileLinkInputStream.read(FileLink.java:171) at org.apache.hadoop.fs.FSDataInputStream.read(FSDataInputStream.java:97) at org.apache.hadoop.hbase.io.util.BlockIOUtils.preadWithExtra(BlockIOUtils.java:235) at org.apache.hadoop.hbase.io.hfile.HFileBlock$FSReaderImpl.readAtOffset(HFileBlock.java:1564) at org.apache.hadoop.hbase.io.hfile.HFileBlock$FSReaderImpl.readBlockDataInternal(HFileBlock.java:1788) at org.apache.hadoop.hbase.io.hfile.HFileBlock$FSReaderImpl.readBlockData(HFileBlock.java:1597) at org.apache.hadoop.hbase.io.hfile.HFileReaderImpl.readBlock(HFileReaderImpl.java:1347) at org.apache.hadoop.hbase.io.hfile.HFileReaderImpl.readBlock(HFileReaderImpl.java:1268) at org.apache.hadoop.hbase.io.hfile.HFileReaderImpl$HFileScannerImpl.readNextDataBlock(HFileReaderImpl.java:762) at org.apache.hadoop.hbase.io.hfile.HFileReaderImpl$EncodedScanner.next(HFileReaderImpl.java:1540) at org.apache.hadoop.hbase.regionserver.StoreFileScanner.reseekAtOrAfter(StoreFileScanner.java:351) at org.apache.hadoop.hbase.regionserver.StoreFileScanner.reseek(StoreFileScanner.java:244) at org.apache.hadoop.hbase.regionserver.NonLazyKeyValueScanner.doRealSeek(NonLazyKeyValueScanner.java:55) at org.apache.hadoop.hbase.regionserver.KeyValueHeap.generalizedSeek(KeyValueHeap.java:322) at org.apache.hadoop.hbase.regionserver.KeyValueHeap.reseek(KeyValueHeap.java:265) at org.apache.hadoop.hbase.regionserver.StoreScanner.reseek(StoreScanner.java:1095) at org.apache.hadoop.hbase.regionserver.StoreScanner.seekAsDirection(StoreScanner.java:1084) at org.apache.hadoop.hbase.regionserver.StoreScanner.seekOrSkipToNextColumn(StoreScanner.java:821) at org.apache.hadoop.hbase.regionserver.StoreScanner.next(StoreScanner.java:684) at org.apache.hadoop.hbase.regionserver.KeyValueHeap.next(KeyValueHeap.java:155) at org.apache.hadoop.hbase.regionserver.HRegion$RegionScannerImpl.populateResult(HRegion.java:7399) at org.apache.hadoop.hbase.regionserver.HRegion$RegionScannerImpl.nextInternal(HRegion.java:7567) at org.apache.hadoop.hbase.regionserver.HRegion$RegionScannerImpl.nextRaw(HRegion.java:7331) at org.apache.hadoop.hbase.regionserver.HRegion$RegionScannerImpl.nextRaw(HRegion.java:7317) at org.apache.phoenix.coprocessor.PagedRegionScanner.next(PagedRegionScanner.java:57) at org.apache.phoenix.coprocessor.PagedRegionScanner.nextRaw(PagedRegionScanner.java:96) at org.apache.phoenix.iterate.RegionScannerFactory$1.nextRaw(RegionScannerFactory.java:211) at org.apache.phoenix.coprocessor.DelegateRegionScanner.nextRaw(DelegateRegionScanner.java:77) at org.apache.phoenix.coprocessor.DelegateRegionScanner.nextRaw(DelegateRegionScanner.java:77) at org.apache.phoenix.coprocessor.BaseScannerRegionObserver$RegionScannerHolder.nextRaw(BaseScannerRegionObserver.java:308) at org.apache.hadoop.hbase.regionserver.RSRpcServices.scan(RSRpcServices.java:3373) - locked <0x00007fc85f0378c0> (a org.apache.phoenix.coprocessor.BaseScannerRegionObserver$RegionScannerHolder) at org.apache.hadoop.hbase.regionserver.RSRpcServices.scan(RSRpcServices.java:3637) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45819) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:392) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:140) at org.apache.hadoop.hbase.ipc.RpcExecutor$Handler.run(RpcExecutor.java:359) at org.apache.hadoop.hbase.ipc.RpcExecutor$Handler.run(RpcExecutor.java:339) "RpcServer.default.FPBQ.Fifo.handler=47,queue=7,port=16020" #134 daemon prio=5 os_prio=0 tid=0x00007fc11d84e800 nid=0x73fb waiting on condition [0x00007fc114bde000] java.lang.Thread.State: WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007fc1c9feb4b0> (a java.util.concurrent.Semaphore$NonfairSync) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer.parkAndCheckInterrupt(AbstractQueuedSynchronizer.java:836) at java.util.concurrent.locks.AbstractQueuedSynchronizer.doAcquireSharedInterruptibly(AbstractQueuedSynchronizer.java:997) at java.util.concurrent.locks.AbstractQueuedSynchronizer.acquireSharedInterruptibly(AbstractQueuedSynchronizer.java:1304) at java.util.concurrent.Semaphore.acquire(Semaphore.java:312) at org.apache.hadoop.hbase.ipc.FastPathBalancedQueueRpcExecutor$FastPathHandler.getCallRunner(FastPathBalancedQueueRpcExecutor.java:109) at org.apache.hadoop.hbase.ipc.RpcExecutor$Handler.run(RpcExecutor.java:339) "RpcServer.default.FPBQ.Fifo.handler=46,queue=6,port=16020" #133 daemon prio=5 os_prio=0 tid=0x00007fc11d84c800 nid=0x73fa waiting for monitor entry [0x00007fc114cde000] java.lang.Thread.State: BLOCKED (on object monitor) at org.apache.hadoop.fs.FSInputStream.read(FSInputStream.java:73) - waiting to lock <0x00007fc4c2769660> (a org.apache.hadoop.fs.s3a.S3AInputStream) at org.apache.hadoop.fs.FSDataInputStream.read(FSDataInputStream.java:97) at org.apache.hadoop.hbase.io.FileLink$FileLinkInputStream.read(FileLink.java:171) at org.apache.hadoop.fs.FSDataInputStream.read(FSDataInputStream.java:97) at org.apache.hadoop.hbase.io.util.BlockIOUtils.preadWithExtra(BlockIOUtils.java:235) at org.apache.hadoop.hbase.io.hfile.HFileBlock$FSReaderImpl.readAtOffset(HFileBlock.java:1564) at org.apache.hadoop.hbase.io.hfile.HFileBlock$FSReaderImpl.readBlockDataInternal(HFileBlock.java:1788) at org.apache.hadoop.hbase.io.hfile.HFileBlock$FSReaderImpl.readBlockData(HFileBlock.java:1597) at org.apache.hadoop.hbase.io.hfile.HFileReaderImpl.readBlock(HFileReaderImpl.java:1347) at org.apache.hadoop.hbase.io.hfile.HFileReaderImpl.readBlock(HFileReaderImpl.java:1268) at org.apache.hadoop.hbase.io.hfile.HFileReaderImpl$HFileScannerImpl.readNextDataBlock(HFileReaderImpl.java:762) at org.apache.hadoop.hbase.io.hfile.HFileReaderImpl$EncodedScanner.next(HFileReaderImpl.java:1540) at org.apache.hadoop.hbase.regionserver.StoreFileScanner.reseekAtOrAfter(StoreFileScanner.java:351) at org.apache.hadoop.hbase.regionserver.StoreFileScanner.reseek(StoreFileScanner.java:244) at org.apache.hadoop.hbase.regionserver.NonLazyKeyValueScanner.doRealSeek(NonLazyKeyValueScanner.java:55) at org.apache.hadoop.hbase.regionserver.KeyValueHeap.generalizedSeek(KeyValueHeap.java:322) at org.apache.hadoop.hbase.regionserver.KeyValueHeap.reseek(KeyValueHeap.java:265) at org.apache.hadoop.hbase.regionserver.StoreScanner.reseek(StoreScanner.java:1095) at org.apache.hadoop.hbase.regionserver.StoreScanner.seekAsDirection(StoreScanner.java:1084) at org.apache.hadoop.hbase.regionserver.StoreScanner.seekOrSkipToNextColumn(StoreScanner.java:821) at org.apache.hadoop.hbase.regionserver.StoreScanner.next(StoreScanner.java:684) at org.apache.hadoop.hbase.regionserver.KeyValueHeap.next(KeyValueHeap.java:155) at org.apache.hadoop.hbase.regionserver.HRegion$RegionScannerImpl.populateResult(HRegion.java:7399) at org.apache.hadoop.hbase.regionserver.HRegion$RegionScannerImpl.nextInternal(HRegion.java:7567) at org.apache.hadoop.hbase.regionserver.HRegion$RegionScannerImpl.nextRaw(HRegion.java:7331) at org.apache.hadoop.hbase.regionserver.HRegion$RegionScannerImpl.nextRaw(HRegion.java:7317) at org.apache.phoenix.coprocessor.PagedRegionScanner.next(PagedRegionScanner.java:57) at org.apache.phoenix.coprocessor.PagedRegionScanner.nextRaw(PagedRegionScanner.java:96) at org.apache.phoenix.iterate.RegionScannerFactory$1.nextRaw(RegionScannerFactory.java:211) at org.apache.phoenix.coprocessor.DelegateRegionScanner.nextRaw(DelegateRegionScanner.java:77) at org.apache.phoenix.coprocessor.DelegateRegionScanner.nextRaw(DelegateRegionScanner.java:77) at org.apache.phoenix.coprocessor.BaseScannerRegionObserver$RegionScannerHolder.nextRaw(BaseScannerRegionObserver.java:308) at org.apache.hadoop.hbase.regionserver.RSRpcServices.scan(RSRpcServices.java:3373) - locked <0x00007fc91d574238> (a org.apache.phoenix.coprocessor.BaseScannerRegionObserver$RegionScannerHolder) at org.apache.hadoop.hbase.regionserver.RSRpcServices.scan(RSRpcServices.java:3637) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45819) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:392) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:140) at org.apache.hadoop.hbase.ipc.RpcExecutor$Handler.run(RpcExecutor.java:359) at org.apache.hadoop.hbase.ipc.RpcExecutor$Handler.run(RpcExecutor.java:339) "RpcServer.default.FPBQ.Fifo.handler=45,queue=5,port=16020" #132 daemon prio=5 os_prio=0 tid=0x00007fc11d84a800 nid=0x73f9 waiting on condition [0x00007fc114de0000] java.lang.Thread.State: WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007fc1c9ff0800> (a java.util.concurrent.Semaphore$NonfairSync) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer.parkAndCheckInterrupt(AbstractQueuedSynchronizer.java:836) at java.util.concurrent.locks.AbstractQueuedSynchronizer.doAcquireSharedInterruptibly(AbstractQueuedSynchronizer.java:997) at java.util.concurrent.locks.AbstractQueuedSynchronizer.acquireSharedInterruptibly(AbstractQueuedSynchronizer.java:1304) at java.util.concurrent.Semaphore.acquire(Semaphore.java:312) at org.apache.hadoop.hbase.ipc.FastPathBalancedQueueRpcExecutor$FastPathHandler.getCallRunner(FastPathBalancedQueueRpcExecutor.java:109) at org.apache.hadoop.hbase.ipc.RpcExecutor$Handler.run(RpcExecutor.java:339) "RpcServer.default.FPBQ.Fifo.handler=44,queue=4,port=16020" #131 daemon prio=5 os_prio=0 tid=0x00007fc11d848000 nid=0x73f8 waiting for monitor entry [0x00007fc114ee0000] java.lang.Thread.State: BLOCKED (on object monitor) at org.apache.hadoop.fs.FSInputStream.read(FSInputStream.java:73) - waiting to lock <0x00007fc4c2769660> (a org.apache.hadoop.fs.s3a.S3AInputStream) at org.apache.hadoop.fs.FSDataInputStream.read(FSDataInputStream.java:97) at org.apache.hadoop.hbase.io.FileLink$FileLinkInputStream.read(FileLink.java:171) at org.apache.hadoop.fs.FSDataInputStream.read(FSDataInputStream.java:97) at org.apache.hadoop.hbase.io.util.BlockIOUtils.preadWithExtra(BlockIOUtils.java:235) at org.apache.hadoop.hbase.io.hfile.HFileBlock$FSReaderImpl.readAtOffset(HFileBlock.java:1564) at org.apache.hadoop.hbase.io.hfile.HFileBlock$FSReaderImpl.readBlockDataInternal(HFileBlock.java:1788) at org.apache.hadoop.hbase.io.hfile.HFileBlock$FSReaderImpl.readBlockData(HFileBlock.java:1597) at org.apache.hadoop.hbase.io.hfile.HFileReaderImpl.readBlock(HFileReaderImpl.java:1347) at org.apache.hadoop.hbase.io.hfile.HFileReaderImpl.readBlock(HFileReaderImpl.java:1268) at org.apache.hadoop.hbase.io.hfile.HFileReaderImpl$HFileScannerImpl.readNextDataBlock(HFileReaderImpl.java:762) at org.apache.hadoop.hbase.io.hfile.HFileReaderImpl$EncodedScanner.next(HFileReaderImpl.java:1540) at org.apache.hadoop.hbase.regionserver.StoreFileScanner.reseekAtOrAfter(StoreFileScanner.java:351) at org.apache.hadoop.hbase.regionserver.StoreFileScanner.reseek(StoreFileScanner.java:244) at org.apache.hadoop.hbase.regionserver.NonLazyKeyValueScanner.doRealSeek(NonLazyKeyValueScanner.java:55) at org.apache.hadoop.hbase.regionserver.KeyValueHeap.generalizedSeek(KeyValueHeap.java:322) at org.apache.hadoop.hbase.regionserver.KeyValueHeap.reseek(KeyValueHeap.java:265) at org.apache.hadoop.hbase.regionserver.StoreScanner.reseek(StoreScanner.java:1095) at org.apache.hadoop.hbase.regionserver.StoreScanner.seekAsDirection(StoreScanner.java:1084) at org.apache.hadoop.hbase.regionserver.StoreScanner.seekOrSkipToNextColumn(StoreScanner.java:821) at org.apache.hadoop.hbase.regionserver.StoreScanner.next(StoreScanner.java:684) at org.apache.hadoop.hbase.regionserver.KeyValueHeap.next(KeyValueHeap.java:155) at org.apache.hadoop.hbase.regionserver.HRegion$RegionScannerImpl.populateResult(HRegion.java:7399) at org.apache.hadoop.hbase.regionserver.HRegion$RegionScannerImpl.nextInternal(HRegion.java:7567) at org.apache.hadoop.hbase.regionserver.HRegion$RegionScannerImpl.nextRaw(HRegion.java:7331) at org.apache.hadoop.hbase.regionserver.HRegion$RegionScannerImpl.nextRaw(HRegion.java:7317) at org.apache.phoenix.coprocessor.PagedRegionScanner.next(PagedRegionScanner.java:57) at org.apache.phoenix.coprocessor.PagedRegionScanner.nextRaw(PagedRegionScanner.java:96) at org.apache.phoenix.iterate.RegionScannerFactory$1.nextRaw(RegionScannerFactory.java:211) at org.apache.phoenix.coprocessor.DelegateRegionScanner.nextRaw(DelegateRegionScanner.java:77) at org.apache.phoenix.coprocessor.DelegateRegionScanner.nextRaw(DelegateRegionScanner.java:77) at org.apache.phoenix.coprocessor.BaseScannerRegionObserver$RegionScannerHolder.nextRaw(BaseScannerRegionObserver.java:308) at org.apache.hadoop.hbase.regionserver.RSRpcServices.scan(RSRpcServices.java:3373) - locked <0x00007fc85f07ba10> (a org.apache.phoenix.coprocessor.BaseScannerRegionObserver$RegionScannerHolder) at org.apache.hadoop.hbase.regionserver.RSRpcServices.scan(RSRpcServices.java:3637) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45819) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:392) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:140) at org.apache.hadoop.hbase.ipc.RpcExecutor$Handler.run(RpcExecutor.java:359) at org.apache.hadoop.hbase.ipc.RpcExecutor$Handler.run(RpcExecutor.java:339) "RpcServer.default.FPBQ.Fifo.handler=43,queue=3,port=16020" #130 daemon prio=5 os_prio=0 tid=0x00007fc11d846000 nid=0x73f7 waiting for monitor entry [0x00007fc114fe1000] java.lang.Thread.State: BLOCKED (on object monitor) at org.apache.hadoop.fs.FSInputStream.read(FSInputStream.java:73) - waiting to lock <0x00007fc4c2769660> (a org.apache.hadoop.fs.s3a.S3AInputStream) at org.apache.hadoop.fs.FSDataInputStream.read(FSDataInputStream.java:97) at org.apache.hadoop.hbase.io.FileLink$FileLinkInputStream.read(FileLink.java:171) at org.apache.hadoop.fs.FSDataInputStream.read(FSDataInputStream.java:97) at org.apache.hadoop.hbase.io.util.BlockIOUtils.preadWithExtra(BlockIOUtils.java:235) at org.apache.hadoop.hbase.io.hfile.HFileBlock$FSReaderImpl.readAtOffset(HFileBlock.java:1564) at org.apache.hadoop.hbase.io.hfile.HFileBlock$FSReaderImpl.readBlockDataInternal(HFileBlock.java:1788) at org.apache.hadoop.hbase.io.hfile.HFileBlock$FSReaderImpl.readBlockData(HFileBlock.java:1597) at org.apache.hadoop.hbase.io.hfile.HFileReaderImpl.readBlock(HFileReaderImpl.java:1347) at org.apache.hadoop.hbase.io.hfile.HFileReaderImpl.readBlock(HFileReaderImpl.java:1268) at org.apache.hadoop.hbase.io.hfile.HFileReaderImpl$HFileScannerImpl.readNextDataBlock(HFileReaderImpl.java:762) at org.apache.hadoop.hbase.io.hfile.HFileReaderImpl$EncodedScanner.next(HFileReaderImpl.java:1540) at org.apache.hadoop.hbase.regionserver.StoreFileScanner.reseekAtOrAfter(StoreFileScanner.java:351) at org.apache.hadoop.hbase.regionserver.StoreFileScanner.reseek(StoreFileScanner.java:244) at org.apache.hadoop.hbase.regionserver.NonLazyKeyValueScanner.doRealSeek(NonLazyKeyValueScanner.java:55) at org.apache.hadoop.hbase.regionserver.KeyValueHeap.generalizedSeek(KeyValueHeap.java:322) at org.apache.hadoop.hbase.regionserver.KeyValueHeap.reseek(KeyValueHeap.java:265) at org.apache.hadoop.hbase.regionserver.StoreScanner.reseek(StoreScanner.java:1095) at org.apache.hadoop.hbase.regionserver.StoreScanner.seekAsDirection(StoreScanner.java:1084) at org.apache.hadoop.hbase.regionserver.StoreScanner.seekOrSkipToNextColumn(StoreScanner.java:821) at org.apache.hadoop.hbase.regionserver.StoreScanner.next(StoreScanner.java:684) at org.apache.hadoop.hbase.regionserver.KeyValueHeap.next(KeyValueHeap.java:155) at org.apache.hadoop.hbase.regionserver.HRegion$RegionScannerImpl.populateResult(HRegion.java:7399) at org.apache.hadoop.hbase.regionserver.HRegion$RegionScannerImpl.nextInternal(HRegion.java:7567) at org.apache.hadoop.hbase.regionserver.HRegion$RegionScannerImpl.nextRaw(HRegion.java:7331) at org.apache.hadoop.hbase.regionserver.HRegion$RegionScannerImpl.nextRaw(HRegion.java:7317) at org.apache.phoenix.coprocessor.PagedRegionScanner.next(PagedRegionScanner.java:57) at org.apache.phoenix.coprocessor.PagedRegionScanner.nextRaw(PagedRegionScanner.java:96) at org.apache.phoenix.iterate.RegionScannerFactory$1.nextRaw(RegionScannerFactory.java:211) at org.apache.phoenix.coprocessor.DelegateRegionScanner.nextRaw(DelegateRegionScanner.java:77) at org.apache.phoenix.coprocessor.DelegateRegionScanner.nextRaw(DelegateRegionScanner.java:77) at org.apache.phoenix.coprocessor.BaseScannerRegionObserver$RegionScannerHolder.nextRaw(BaseScannerRegionObserver.java:308) at org.apache.hadoop.hbase.regionserver.RSRpcServices.scan(RSRpcServices.java:3373) - locked <0x00007fc85f062c00> (a org.apache.phoenix.coprocessor.BaseScannerRegionObserver$RegionScannerHolder) at org.apache.hadoop.hbase.regionserver.RSRpcServices.scan(RSRpcServices.java:3637) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45819) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:392) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:140) at org.apache.hadoop.hbase.ipc.RpcExecutor$Handler.run(RpcExecutor.java:359) at org.apache.hadoop.hbase.ipc.RpcExecutor$Handler.run(RpcExecutor.java:339) "RpcServer.default.FPBQ.Fifo.handler=42,queue=2,port=16020" #129 daemon prio=5 os_prio=0 tid=0x00007fc11d844000 nid=0x73f6 waiting for monitor entry [0x00007fc1150e2000] java.lang.Thread.State: BLOCKED (on object monitor) at org.apache.hadoop.fs.FSInputStream.read(FSInputStream.java:73) - waiting to lock <0x00007fc4c2769660> (a org.apache.hadoop.fs.s3a.S3AInputStream) at org.apache.hadoop.fs.FSDataInputStream.read(FSDataInputStream.java:97) at org.apache.hadoop.hbase.io.FileLink$FileLinkInputStream.read(FileLink.java:171) at org.apache.hadoop.fs.FSDataInputStream.read(FSDataInputStream.java:97) at org.apache.hadoop.hbase.io.util.BlockIOUtils.preadWithExtra(BlockIOUtils.java:235) at org.apache.hadoop.hbase.io.hfile.HFileBlock$FSReaderImpl.readAtOffset(HFileBlock.java:1564) at org.apache.hadoop.hbase.io.hfile.HFileBlock$FSReaderImpl.readBlockDataInternal(HFileBlock.java:1788) at org.apache.hadoop.hbase.io.hfile.HFileBlock$FSReaderImpl.readBlockData(HFileBlock.java:1597) at org.apache.hadoop.hbase.io.hfile.HFileReaderImpl.readBlock(HFileReaderImpl.java:1347) at org.apache.hadoop.hbase.io.hfile.HFileReaderImpl.readBlock(HFileReaderImpl.java:1268) at org.apache.hadoop.hbase.io.hfile.HFileReaderImpl$HFileScannerImpl.readNextDataBlock(HFileReaderImpl.java:762) at org.apache.hadoop.hbase.io.hfile.HFileReaderImpl$EncodedScanner.next(HFileReaderImpl.java:1540) at org.apache.hadoop.hbase.regionserver.StoreFileScanner.reseekAtOrAfter(StoreFileScanner.java:351) at org.apache.hadoop.hbase.regionserver.StoreFileScanner.reseek(StoreFileScanner.java:244) at org.apache.hadoop.hbase.regionserver.NonLazyKeyValueScanner.doRealSeek(NonLazyKeyValueScanner.java:55) at org.apache.hadoop.hbase.regionserver.KeyValueHeap.generalizedSeek(KeyValueHeap.java:322) at org.apache.hadoop.hbase.regionserver.KeyValueHeap.reseek(KeyValueHeap.java:265) at org.apache.hadoop.hbase.regionserver.StoreScanner.reseek(StoreScanner.java:1095) at org.apache.hadoop.hbase.regionserver.StoreScanner.seekAsDirection(StoreScanner.java:1084) at org.apache.hadoop.hbase.regionserver.StoreScanner.seekOrSkipToNextColumn(StoreScanner.java:821) at org.apache.hadoop.hbase.regionserver.StoreScanner.next(StoreScanner.java:684) at org.apache.hadoop.hbase.regionserver.KeyValueHeap.next(KeyValueHeap.java:155) at org.apache.hadoop.hbase.regionserver.HRegion$RegionScannerImpl.populateResult(HRegion.java:7399) at org.apache.hadoop.hbase.regionserver.HRegion$RegionScannerImpl.nextInternal(HRegion.java:7567) at org.apache.hadoop.hbase.regionserver.HRegion$RegionScannerImpl.nextRaw(HRegion.java:7331) at org.apache.hadoop.hbase.regionserver.HRegion$RegionScannerImpl.nextRaw(HRegion.java:7317) at org.apache.phoenix.coprocessor.PagedRegionScanner.next(PagedRegionScanner.java:57) at org.apache.phoenix.coprocessor.PagedRegionScanner.nextRaw(PagedRegionScanner.java:96) at org.apache.phoenix.iterate.RegionScannerFactory$1.nextRaw(RegionScannerFactory.java:211) at org.apache.phoenix.coprocessor.DelegateRegionScanner.nextRaw(DelegateRegionScanner.java:77) at org.apache.phoenix.coprocessor.DelegateRegionScanner.nextRaw(DelegateRegionScanner.java:77) at org.apache.phoenix.coprocessor.BaseScannerRegionObserver$RegionScannerHolder.nextRaw(BaseScannerRegionObserver.java:308) at org.apache.hadoop.hbase.regionserver.RSRpcServices.scan(RSRpcServices.java:3373) - locked <0x00007fc85f037ec8> (a org.apache.phoenix.coprocessor.BaseScannerRegionObserver$RegionScannerHolder) at org.apache.hadoop.hbase.regionserver.RSRpcServices.scan(RSRpcServices.java:3637) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45819) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:392) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:140) at org.apache.hadoop.hbase.ipc.RpcExecutor$Handler.run(RpcExecutor.java:359) at org.apache.hadoop.hbase.ipc.RpcExecutor$Handler.run(RpcExecutor.java:339) "RpcServer.default.FPBQ.Fifo.handler=41,queue=1,port=16020" #128 daemon prio=5 os_prio=0 tid=0x00007fc11d842000 nid=0x73f5 waiting for monitor entry [0x00007fc1151e3000] java.lang.Thread.State: BLOCKED (on object monitor) at org.apache.hadoop.fs.FSInputStream.read(FSInputStream.java:73) - waiting to lock <0x00007fc4c2769660> (a org.apache.hadoop.fs.s3a.S3AInputStream) at org.apache.hadoop.fs.FSDataInputStream.read(FSDataInputStream.java:97) at org.apache.hadoop.hbase.io.FileLink$FileLinkInputStream.read(FileLink.java:171) at org.apache.hadoop.fs.FSDataInputStream.read(FSDataInputStream.java:97) at org.apache.hadoop.hbase.io.util.BlockIOUtils.preadWithExtra(BlockIOUtils.java:235) at org.apache.hadoop.hbase.io.hfile.HFileBlock$FSReaderImpl.readAtOffset(HFileBlock.java:1564) at org.apache.hadoop.hbase.io.hfile.HFileBlock$FSReaderImpl.readBlockDataInternal(HFileBlock.java:1788) at org.apache.hadoop.hbase.io.hfile.HFileBlock$FSReaderImpl.readBlockData(HFileBlock.java:1597) at org.apache.hadoop.hbase.io.hfile.HFileReaderImpl.readBlock(HFileReaderImpl.java:1347) at org.apache.hadoop.hbase.io.hfile.HFileReaderImpl.readBlock(HFileReaderImpl.java:1268) at org.apache.hadoop.hbase.io.hfile.HFileReaderImpl$HFileScannerImpl.readNextDataBlock(HFileReaderImpl.java:762) at org.apache.hadoop.hbase.io.hfile.HFileReaderImpl$EncodedScanner.next(HFileReaderImpl.java:1540) at org.apache.hadoop.hbase.regionserver.StoreFileScanner.reseekAtOrAfter(StoreFileScanner.java:351) at org.apache.hadoop.hbase.regionserver.StoreFileScanner.reseek(StoreFileScanner.java:244) at org.apache.hadoop.hbase.regionserver.NonLazyKeyValueScanner.doRealSeek(NonLazyKeyValueScanner.java:55) at org.apache.hadoop.hbase.regionserver.KeyValueHeap.generalizedSeek(KeyValueHeap.java:322) at org.apache.hadoop.hbase.regionserver.KeyValueHeap.reseek(KeyValueHeap.java:265) at org.apache.hadoop.hbase.regionserver.StoreScanner.reseek(StoreScanner.java:1095) at org.apache.hadoop.hbase.regionserver.StoreScanner.seekAsDirection(StoreScanner.java:1084) at org.apache.hadoop.hbase.regionserver.StoreScanner.seekOrSkipToNextColumn(StoreScanner.java:821) at org.apache.hadoop.hbase.regionserver.StoreScanner.next(StoreScanner.java:684) at org.apache.hadoop.hbase.regionserver.KeyValueHeap.next(KeyValueHeap.java:155) at org.apache.hadoop.hbase.regionserver.HRegion$RegionScannerImpl.populateResult(HRegion.java:7399) at org.apache.hadoop.hbase.regionserver.HRegion$RegionScannerImpl.nextInternal(HRegion.java:7567) at org.apache.hadoop.hbase.regionserver.HRegion$RegionScannerImpl.nextRaw(HRegion.java:7331) at org.apache.hadoop.hbase.regionserver.HRegion$RegionScannerImpl.nextRaw(HRegion.java:7317) at org.apache.phoenix.coprocessor.PagedRegionScanner.next(PagedRegionScanner.java:57) at org.apache.phoenix.coprocessor.PagedRegionScanner.nextRaw(PagedRegionScanner.java:96) at org.apache.phoenix.iterate.RegionScannerFactory$1.nextRaw(RegionScannerFactory.java:211) at org.apache.phoenix.coprocessor.DelegateRegionScanner.nextRaw(DelegateRegionScanner.java:77) at org.apache.phoenix.coprocessor.DelegateRegionScanner.nextRaw(DelegateRegionScanner.java:77) at org.apache.phoenix.coprocessor.BaseScannerRegionObserver$RegionScannerHolder.nextRaw(BaseScannerRegionObserver.java:308) at org.apache.hadoop.hbase.regionserver.RSRpcServices.scan(RSRpcServices.java:3373) - locked <0x00007fc85f02fa40> (a org.apache.phoenix.coprocessor.BaseScannerRegionObserver$RegionScannerHolder) at org.apache.hadoop.hbase.regionserver.RSRpcServices.scan(RSRpcServices.java:3637) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45819) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:392) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:140) at org.apache.hadoop.hbase.ipc.RpcExecutor$Handler.run(RpcExecutor.java:359) at org.apache.hadoop.hbase.ipc.RpcExecutor$Handler.run(RpcExecutor.java:339) "RpcServer.default.FPBQ.Fifo.handler=40,queue=0,port=16020" #127 daemon prio=5 os_prio=0 tid=0x00007fc11d840000 nid=0x73f4 waiting for monitor entry [0x00007fc1152e4000] java.lang.Thread.State: BLOCKED (on object monitor) at org.apache.hadoop.fs.FSInputStream.read(FSInputStream.java:73) - waiting to lock <0x00007fc4c2769660> (a org.apache.hadoop.fs.s3a.S3AInputStream) at org.apache.hadoop.fs.FSDataInputStream.read(FSDataInputStream.java:97) at org.apache.hadoop.hbase.io.FileLink$FileLinkInputStream.read(FileLink.java:171) at org.apache.hadoop.fs.FSDataInputStream.read(FSDataInputStream.java:97) at org.apache.hadoop.hbase.io.util.BlockIOUtils.preadWithExtra(BlockIOUtils.java:235) at org.apache.hadoop.hbase.io.hfile.HFileBlock$FSReaderImpl.readAtOffset(HFileBlock.java:1564) at org.apache.hadoop.hbase.io.hfile.HFileBlock$FSReaderImpl.readBlockDataInternal(HFileBlock.java:1788) at org.apache.hadoop.hbase.io.hfile.HFileBlock$FSReaderImpl.readBlockData(HFileBlock.java:1597) at org.apache.hadoop.hbase.io.hfile.HFileReaderImpl.readBlock(HFileReaderImpl.java:1347) at org.apache.hadoop.hbase.io.hfile.HFileReaderImpl.readBlock(HFileReaderImpl.java:1268) at org.apache.hadoop.hbase.io.hfile.HFileReaderImpl$HFileScannerImpl.readNextDataBlock(HFileReaderImpl.java:762) at org.apache.hadoop.hbase.io.hfile.HFileReaderImpl$EncodedScanner.next(HFileReaderImpl.java:1540) at org.apache.hadoop.hbase.regionserver.StoreFileScanner.reseekAtOrAfter(StoreFileScanner.java:351) at org.apache.hadoop.hbase.regionserver.StoreFileScanner.reseek(StoreFileScanner.java:244) at org.apache.hadoop.hbase.regionserver.NonLazyKeyValueScanner.doRealSeek(NonLazyKeyValueScanner.java:55) at org.apache.hadoop.hbase.regionserver.KeyValueHeap.generalizedSeek(KeyValueHeap.java:322) at org.apache.hadoop.hbase.regionserver.KeyValueHeap.reseek(KeyValueHeap.java:265) at org.apache.hadoop.hbase.regionserver.StoreScanner.reseek(StoreScanner.java:1095) at org.apache.hadoop.hbase.regionserver.StoreScanner.seekAsDirection(StoreScanner.java:1084) at org.apache.hadoop.hbase.regionserver.StoreScanner.seekOrSkipToNextColumn(StoreScanner.java:821) at org.apache.hadoop.hbase.regionserver.StoreScanner.next(StoreScanner.java:684) at org.apache.hadoop.hbase.regionserver.KeyValueHeap.next(KeyValueHeap.java:155) at org.apache.hadoop.hbase.regionserver.HRegion$RegionScannerImpl.populateResult(HRegion.java:7399) at org.apache.hadoop.hbase.regionserver.HRegion$RegionScannerImpl.nextInternal(HRegion.java:7567) at org.apache.hadoop.hbase.regionserver.HRegion$RegionScannerImpl.nextRaw(HRegion.java:7331) at org.apache.hadoop.hbase.regionserver.HRegion$RegionScannerImpl.nextRaw(HRegion.java:7317) at org.apache.phoenix.coprocessor.PagedRegionScanner.next(PagedRegionScanner.java:57) at org.apache.phoenix.coprocessor.PagedRegionScanner.nextRaw(PagedRegionScanner.java:96) at org.apache.phoenix.iterate.RegionScannerFactory$1.nextRaw(RegionScannerFactory.java:211) at org.apache.phoenix.coprocessor.DelegateRegionScanner.nextRaw(DelegateRegionScanner.java:77) at org.apache.phoenix.coprocessor.DelegateRegionScanner.nextRaw(DelegateRegionScanner.java:77) at org.apache.phoenix.coprocessor.BaseScannerRegionObserver$RegionScannerHolder.nextRaw(BaseScannerRegionObserver.java:308) at org.apache.hadoop.hbase.regionserver.RSRpcServices.scan(RSRpcServices.java:3373) - locked <0x00007fc85f07db48> (a org.apache.phoenix.coprocessor.BaseScannerRegionObserver$RegionScannerHolder) at org.apache.hadoop.hbase.regionserver.RSRpcServices.scan(RSRpcServices.java:3637) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45819) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:392) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:140) at org.apache.hadoop.hbase.ipc.RpcExecutor$Handler.run(RpcExecutor.java:359) at org.apache.hadoop.hbase.ipc.RpcExecutor$Handler.run(RpcExecutor.java:339) "RpcServer.default.FPBQ.Fifo.handler=39,queue=9,port=16020" #126 daemon prio=5 os_prio=0 tid=0x00007fc11d83e000 nid=0x73f3 waiting for monitor entry [0x00007fc1153e5000] java.lang.Thread.State: BLOCKED (on object monitor) at org.apache.hadoop.fs.FSInputStream.read(FSInputStream.java:73) - waiting to lock <0x00007fc4c2769660> (a org.apache.hadoop.fs.s3a.S3AInputStream) at org.apache.hadoop.fs.FSDataInputStream.read(FSDataInputStream.java:97) at org.apache.hadoop.hbase.io.FileLink$FileLinkInputStream.read(FileLink.java:171) at org.apache.hadoop.fs.FSDataInputStream.read(FSDataInputStream.java:97) at org.apache.hadoop.hbase.io.util.BlockIOUtils.preadWithExtra(BlockIOUtils.java:235) at org.apache.hadoop.hbase.io.hfile.HFileBlock$FSReaderImpl.readAtOffset(HFileBlock.java:1564) at org.apache.hadoop.hbase.io.hfile.HFileBlock$FSReaderImpl.readBlockDataInternal(HFileBlock.java:1788) at org.apache.hadoop.hbase.io.hfile.HFileBlock$FSReaderImpl.readBlockData(HFileBlock.java:1597) at org.apache.hadoop.hbase.io.hfile.HFileReaderImpl.readBlock(HFileReaderImpl.java:1347) at org.apache.hadoop.hbase.io.hfile.HFileReaderImpl.readBlock(HFileReaderImpl.java:1268) at org.apache.hadoop.hbase.io.hfile.HFileReaderImpl$HFileScannerImpl.readNextDataBlock(HFileReaderImpl.java:762) at org.apache.hadoop.hbase.io.hfile.HFileReaderImpl$EncodedScanner.next(HFileReaderImpl.java:1540) at org.apache.hadoop.hbase.regionserver.StoreFileScanner.reseekAtOrAfter(StoreFileScanner.java:351) at org.apache.hadoop.hbase.regionserver.StoreFileScanner.reseek(StoreFileScanner.java:244) at org.apache.hadoop.hbase.regionserver.NonLazyKeyValueScanner.doRealSeek(NonLazyKeyValueScanner.java:55) at org.apache.hadoop.hbase.regionserver.KeyValueHeap.generalizedSeek(KeyValueHeap.java:322) at org.apache.hadoop.hbase.regionserver.KeyValueHeap.reseek(KeyValueHeap.java:265) at org.apache.hadoop.hbase.regionserver.StoreScanner.reseek(StoreScanner.java:1095) at org.apache.hadoop.hbase.regionserver.StoreScanner.seekAsDirection(StoreScanner.java:1084) at org.apache.hadoop.hbase.regionserver.StoreScanner.seekOrSkipToNextColumn(StoreScanner.java:821) at org.apache.hadoop.hbase.regionserver.StoreScanner.next(StoreScanner.java:684) at org.apache.hadoop.hbase.regionserver.KeyValueHeap.next(KeyValueHeap.java:155) at org.apache.hadoop.hbase.regionserver.HRegion$RegionScannerImpl.populateResult(HRegion.java:7399) at org.apache.hadoop.hbase.regionserver.HRegion$RegionScannerImpl.nextInternal(HRegion.java:7567) at org.apache.hadoop.hbase.regionserver.HRegion$RegionScannerImpl.nextRaw(HRegion.java:7331) at org.apache.hadoop.hbase.regionserver.HRegion$RegionScannerImpl.nextRaw(HRegion.java:7317) at org.apache.phoenix.coprocessor.PagedRegionScanner.next(PagedRegionScanner.java:57) at org.apache.phoenix.coprocessor.PagedRegionScanner.nextRaw(PagedRegionScanner.java:96) at org.apache.phoenix.iterate.RegionScannerFactory$1.nextRaw(RegionScannerFactory.java:211) at org.apache.phoenix.coprocessor.DelegateRegionScanner.nextRaw(DelegateRegionScanner.java:77) at org.apache.phoenix.coprocessor.DelegateRegionScanner.nextRaw(DelegateRegionScanner.java:77) at org.apache.phoenix.coprocessor.BaseScannerRegionObserver$RegionScannerHolder.nextRaw(BaseScannerRegionObserver.java:308) at org.apache.hadoop.hbase.regionserver.RSRpcServices.scan(RSRpcServices.java:3373) - locked <0x00007fc85f07fc68> (a org.apache.phoenix.coprocessor.BaseScannerRegionObserver$RegionScannerHolder) at org.apache.hadoop.hbase.regionserver.RSRpcServices.scan(RSRpcServices.java:3637) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45819) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:392) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:140) at org.apache.hadoop.hbase.ipc.RpcExecutor$Handler.run(RpcExecutor.java:359) at org.apache.hadoop.hbase.ipc.RpcExecutor$Handler.run(RpcExecutor.java:339) "RpcServer.default.FPBQ.Fifo.handler=38,queue=8,port=16020" #125 daemon prio=5 os_prio=0 tid=0x00007fc11d83c000 nid=0x73f2 waiting for monitor entry [0x00007fc1154e6000] java.lang.Thread.State: BLOCKED (on object monitor) at org.apache.hadoop.fs.FSInputStream.read(FSInputStream.java:73) - waiting to lock <0x00007fc4c2769660> (a org.apache.hadoop.fs.s3a.S3AInputStream) at org.apache.hadoop.fs.FSDataInputStream.read(FSDataInputStream.java:97) at org.apache.hadoop.hbase.io.FileLink$FileLinkInputStream.read(FileLink.java:171) at org.apache.hadoop.fs.FSDataInputStream.read(FSDataInputStream.java:97) at org.apache.hadoop.hbase.io.util.BlockIOUtils.preadWithExtra(BlockIOUtils.java:235) at org.apache.hadoop.hbase.io.hfile.HFileBlock$FSReaderImpl.readAtOffset(HFileBlock.java:1564) at org.apache.hadoop.hbase.io.hfile.HFileBlock$FSReaderImpl.readBlockDataInternal(HFileBlock.java:1788) at org.apache.hadoop.hbase.io.hfile.HFileBlock$FSReaderImpl.readBlockData(HFileBlock.java:1597) at org.apache.hadoop.hbase.io.hfile.HFileReaderImpl.readBlock(HFileReaderImpl.java:1347) at org.apache.hadoop.hbase.io.hfile.HFileReaderImpl.readBlock(HFileReaderImpl.java:1268) at org.apache.hadoop.hbase.io.hfile.HFileReaderImpl$HFileScannerImpl.readNextDataBlock(HFileReaderImpl.java:762) at org.apache.hadoop.hbase.io.hfile.HFileReaderImpl$EncodedScanner.next(HFileReaderImpl.java:1540) at org.apache.hadoop.hbase.regionserver.StoreFileScanner.reseekAtOrAfter(StoreFileScanner.java:351) at org.apache.hadoop.hbase.regionserver.StoreFileScanner.reseek(StoreFileScanner.java:244) at org.apache.hadoop.hbase.regionserver.NonLazyKeyValueScanner.doRealSeek(NonLazyKeyValueScanner.java:55) at org.apache.hadoop.hbase.regionserver.KeyValueHeap.generalizedSeek(KeyValueHeap.java:322) at org.apache.hadoop.hbase.regionserver.KeyValueHeap.reseek(KeyValueHeap.java:265) at org.apache.hadoop.hbase.regionserver.StoreScanner.reseek(StoreScanner.java:1095) at org.apache.hadoop.hbase.regionserver.StoreScanner.seekAsDirection(StoreScanner.java:1084) at org.apache.hadoop.hbase.regionserver.StoreScanner.seekOrSkipToNextColumn(StoreScanner.java:821) at org.apache.hadoop.hbase.regionserver.StoreScanner.next(StoreScanner.java:684) at org.apache.hadoop.hbase.regionserver.KeyValueHeap.next(KeyValueHeap.java:155) at org.apache.hadoop.hbase.regionserver.HRegion$RegionScannerImpl.populateResult(HRegion.java:7399) at org.apache.hadoop.hbase.regionserver.HRegion$RegionScannerImpl.nextInternal(HRegion.java:7567) at org.apache.hadoop.hbase.regionserver.HRegion$RegionScannerImpl.nextRaw(HRegion.java:7331) at org.apache.hadoop.hbase.regionserver.HRegion$RegionScannerImpl.nextRaw(HRegion.java:7317) at org.apache.phoenix.coprocessor.PagedRegionScanner.next(PagedRegionScanner.java:57) at org.apache.phoenix.coprocessor.PagedRegionScanner.nextRaw(PagedRegionScanner.java:96) at org.apache.phoenix.iterate.RegionScannerFactory$1.nextRaw(RegionScannerFactory.java:211) at org.apache.phoenix.coprocessor.DelegateRegionScanner.nextRaw(DelegateRegionScanner.java:77) at org.apache.phoenix.coprocessor.DelegateRegionScanner.nextRaw(DelegateRegionScanner.java:77) at org.apache.phoenix.coprocessor.BaseScannerRegionObserver$RegionScannerHolder.nextRaw(BaseScannerRegionObserver.java:308) at org.apache.hadoop.hbase.regionserver.RSRpcServices.scan(RSRpcServices.java:3373) - locked <0x00007fc85f02ff98> (a org.apache.phoenix.coprocessor.BaseScannerRegionObserver$RegionScannerHolder) at org.apache.hadoop.hbase.regionserver.RSRpcServices.scan(RSRpcServices.java:3637) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45819) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:392) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:140) at org.apache.hadoop.hbase.ipc.RpcExecutor$Handler.run(RpcExecutor.java:359) at org.apache.hadoop.hbase.ipc.RpcExecutor$Handler.run(RpcExecutor.java:339) "RpcServer.default.FPBQ.Fifo.handler=37,queue=7,port=16020" #124 daemon prio=5 os_prio=0 tid=0x00007fc11d83a000 nid=0x73f1 waiting for monitor entry [0x00007fc1155e7000] java.lang.Thread.State: BLOCKED (on object monitor) at org.apache.hadoop.fs.FSInputStream.read(FSInputStream.java:73) - waiting to lock <0x00007fc4c2769660> (a org.apache.hadoop.fs.s3a.S3AInputStream) at org.apache.hadoop.fs.FSDataInputStream.read(FSDataInputStream.java:97) at org.apache.hadoop.hbase.io.FileLink$FileLinkInputStream.read(FileLink.java:171) at org.apache.hadoop.fs.FSDataInputStream.read(FSDataInputStream.java:97) at org.apache.hadoop.hbase.io.util.BlockIOUtils.preadWithExtra(BlockIOUtils.java:235) at org.apache.hadoop.hbase.io.hfile.HFileBlock$FSReaderImpl.readAtOffset(HFileBlock.java:1564) at org.apache.hadoop.hbase.io.hfile.HFileBlock$FSReaderImpl.readBlockDataInternal(HFileBlock.java:1788) at org.apache.hadoop.hbase.io.hfile.HFileBlock$FSReaderImpl.readBlockData(HFileBlock.java:1597) at org.apache.hadoop.hbase.io.hfile.HFileReaderImpl.readBlock(HFileReaderImpl.java:1347) at org.apache.hadoop.hbase.io.hfile.HFileReaderImpl.readBlock(HFileReaderImpl.java:1268) at org.apache.hadoop.hbase.io.hfile.HFileReaderImpl$HFileScannerImpl.readNextDataBlock(HFileReaderImpl.java:762) at org.apache.hadoop.hbase.io.hfile.HFileReaderImpl$EncodedScanner.next(HFileReaderImpl.java:1540) at org.apache.hadoop.hbase.regionserver.StoreFileScanner.reseekAtOrAfter(StoreFileScanner.java:351) at org.apache.hadoop.hbase.regionserver.StoreFileScanner.reseek(StoreFileScanner.java:244) at org.apache.hadoop.hbase.regionserver.NonLazyKeyValueScanner.doRealSeek(NonLazyKeyValueScanner.java:55) at org.apache.hadoop.hbase.regionserver.KeyValueHeap.generalizedSeek(KeyValueHeap.java:322) at org.apache.hadoop.hbase.regionserver.KeyValueHeap.reseek(KeyValueHeap.java:265) at org.apache.hadoop.hbase.regionserver.StoreScanner.reseek(StoreScanner.java:1095) at org.apache.hadoop.hbase.regionserver.StoreScanner.seekAsDirection(StoreScanner.java:1084) at org.apache.hadoop.hbase.regionserver.StoreScanner.seekOrSkipToNextColumn(StoreScanner.java:821) at org.apache.hadoop.hbase.regionserver.StoreScanner.next(StoreScanner.java:684) at org.apache.hadoop.hbase.regionserver.KeyValueHeap.next(KeyValueHeap.java:155) at org.apache.hadoop.hbase.regionserver.HRegion$RegionScannerImpl.populateResult(HRegion.java:7399) at org.apache.hadoop.hbase.regionserver.HRegion$RegionScannerImpl.nextInternal(HRegion.java:7567) at org.apache.hadoop.hbase.regionserver.HRegion$RegionScannerImpl.nextRaw(HRegion.java:7331) at org.apache.hadoop.hbase.regionserver.HRegion$RegionScannerImpl.nextRaw(HRegion.java:7317) at org.apache.phoenix.coprocessor.PagedRegionScanner.next(PagedRegionScanner.java:57) at org.apache.phoenix.coprocessor.PagedRegionScanner.nextRaw(PagedRegionScanner.java:96) at org.apache.phoenix.iterate.RegionScannerFactory$1.nextRaw(RegionScannerFactory.java:211) at org.apache.phoenix.coprocessor.DelegateRegionScanner.nextRaw(DelegateRegionScanner.java:77) at org.apache.phoenix.coprocessor.DelegateRegionScanner.nextRaw(DelegateRegionScanner.java:77) at org.apache.phoenix.coprocessor.BaseScannerRegionObserver$RegionScannerHolder.nextRaw(BaseScannerRegionObserver.java:308) at org.apache.hadoop.hbase.regionserver.RSRpcServices.scan(RSRpcServices.java:3373) - locked <0x00007fc6b1397708> (a org.apache.phoenix.coprocessor.BaseScannerRegionObserver$RegionScannerHolder) at org.apache.hadoop.hbase.regionserver.RSRpcServices.scan(RSRpcServices.java:3637) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45819) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:392) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:140) at org.apache.hadoop.hbase.ipc.RpcExecutor$Handler.run(RpcExecutor.java:359) at org.apache.hadoop.hbase.ipc.RpcExecutor$Handler.run(RpcExecutor.java:339) "RpcServer.default.FPBQ.Fifo.handler=36,queue=6,port=16020" #123 daemon prio=5 os_prio=0 tid=0x00007fc11d838000 nid=0x73f0 runnable [0x00007fc1156e8000] java.lang.Thread.State: RUNNABLE at java.net.SocketInputStream.socketRead0(Native Method) at java.net.SocketInputStream.socketRead(SocketInputStream.java:116) at java.net.SocketInputStream.read(SocketInputStream.java:171) at java.net.SocketInputStream.read(SocketInputStream.java:141) at sun.security.ssl.SSLSocketInputRecord.read(SSLSocketInputRecord.java:464) at sun.security.ssl.SSLSocketInputRecord.bytesInCompletePacket(SSLSocketInputRecord.java:68) at sun.security.ssl.SSLSocketImpl.readApplicationRecord(SSLSocketImpl.java:1346) at sun.security.ssl.SSLSocketImpl.access$300(SSLSocketImpl.java:73) at sun.security.ssl.SSLSocketImpl$AppInputStream.read(SSLSocketImpl.java:962) at com.amazonaws.thirdparty.apache.http.impl.io.SessionInputBufferImpl.streamRead(SessionInputBufferImpl.java:137) at com.amazonaws.thirdparty.apache.http.impl.io.SessionInputBufferImpl.fillBuffer(SessionInputBufferImpl.java:153) at com.amazonaws.thirdparty.apache.http.impl.io.SessionInputBufferImpl.readLine(SessionInputBufferImpl.java:280) at com.amazonaws.thirdparty.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:138) at com.amazonaws.thirdparty.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:56) at com.amazonaws.thirdparty.apache.http.impl.io.AbstractMessageParser.parse(AbstractMessageParser.java:259) at com.amazonaws.thirdparty.apache.http.impl.DefaultBHttpClientConnection.receiveResponseHeader(DefaultBHttpClientConnection.java:163) at com.amazonaws.thirdparty.apache.http.impl.conn.CPoolProxy.receiveResponseHeader(CPoolProxy.java:157) at com.amazonaws.thirdparty.apache.http.protocol.HttpRequestExecutor.doReceiveResponse(HttpRequestExecutor.java:273) at com.amazonaws.http.protocol.SdkHttpRequestExecutor.doReceiveResponse(SdkHttpRequestExecutor.java:82) at com.amazonaws.thirdparty.apache.http.protocol.HttpRequestExecutor.execute(HttpRequestExecutor.java:125) at com.amazonaws.thirdparty.apache.http.impl.execchain.MainClientExec.execute(MainClientExec.java:272) at com.amazonaws.thirdparty.apache.http.impl.execchain.ProtocolExec.execute(ProtocolExec.java:186) at com.amazonaws.thirdparty.apache.http.impl.client.InternalHttpClient.doExecute(InternalHttpClient.java:185) at com.amazonaws.thirdparty.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:83) at com.amazonaws.thirdparty.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:56) at com.amazonaws.http.apache.client.impl.SdkHttpClient.execute(SdkHttpClient.java:72) at com.amazonaws.http.AmazonHttpClient$RequestExecutor.executeOneRequest(AmazonHttpClient.java:1343) at com.amazonaws.http.AmazonHttpClient$RequestExecutor.executeHelper(AmazonHttpClient.java:1154) at com.amazonaws.http.AmazonHttpClient$RequestExecutor.doExecute(AmazonHttpClient.java:811) at com.amazonaws.http.AmazonHttpClient$RequestExecutor.executeWithTimer(AmazonHttpClient.java:779) at com.amazonaws.http.AmazonHttpClient$RequestExecutor.execute(AmazonHttpClient.java:753) at com.amazonaws.http.AmazonHttpClient$RequestExecutor.access$500(AmazonHttpClient.java:713) at com.amazonaws.http.AmazonHttpClient$RequestExecutionBuilderImpl.execute(AmazonHttpClient.java:695) at com.amazonaws.http.AmazonHttpClient.execute(AmazonHttpClient.java:559) at com.amazonaws.http.AmazonHttpClient.execute(AmazonHttpClient.java:539) at com.amazonaws.services.s3.AmazonS3Client.invoke(AmazonS3Client.java:5453) at com.amazonaws.services.s3.AmazonS3Client.invoke(AmazonS3Client.java:5400) at com.amazonaws.services.s3.AmazonS3Client.getObject(AmazonS3Client.java:1524) at org.apache.hadoop.fs.s3a.S3AFileSystem$InputStreamCallbacksImpl.getObject(S3AFileSystem.java:1506) at org.apache.hadoop.fs.s3a.S3AInputStream.lambda$reopen$0(S3AInputStream.java:217) at org.apache.hadoop.fs.s3a.S3AInputStream$$Lambda$327/1995766829.apply(Unknown Source) at org.apache.hadoop.fs.s3a.Invoker.once(Invoker.java:117) at org.apache.hadoop.fs.s3a.S3AInputStream.reopen(S3AInputStream.java:216) - locked <0x00007fc4c2769660> (a org.apache.hadoop.fs.s3a.S3AInputStream) at org.apache.hadoop.fs.s3a.S3AInputStream.lambda$lazySeek$1(S3AInputStream.java:382) at org.apache.hadoop.fs.s3a.S3AInputStream$$Lambda$323/1719986493.apply(Unknown Source) at org.apache.hadoop.fs.s3a.Invoker.lambda$maybeRetry$3(Invoker.java:230) at org.apache.hadoop.fs.s3a.Invoker$$Lambda$324/1109135669.apply(Unknown Source) at org.apache.hadoop.fs.s3a.Invoker.once(Invoker.java:117) at org.apache.hadoop.fs.s3a.Invoker.maybeRetry(Invoker.java:356) at org.apache.hadoop.fs.s3a.Invoker.maybeRetry(Invoker.java:228) at org.apache.hadoop.fs.s3a.Invoker.maybeRetry(Invoker.java:272) at org.apache.hadoop.fs.s3a.S3AInputStream.lazySeek(S3AInputStream.java:374) at org.apache.hadoop.fs.s3a.S3AInputStream.read(S3AInputStream.java:496) - locked <0x00007fc4c2769660> (a org.apache.hadoop.fs.s3a.S3AInputStream) at org.apache.hadoop.fs.FSInputStream.read(FSInputStream.java:78) - locked <0x00007fc4c2769660> (a org.apache.hadoop.fs.s3a.S3AInputStream) at org.apache.hadoop.fs.FSDataInputStream.read(FSDataInputStream.java:97) at org.apache.hadoop.hbase.io.FileLink$FileLinkInputStream.read(FileLink.java:171) at org.apache.hadoop.fs.FSDataInputStream.read(FSDataInputStream.java:97) at org.apache.hadoop.hbase.io.util.BlockIOUtils.preadWithExtra(BlockIOUtils.java:235) at org.apache.hadoop.hbase.io.hfile.HFileBlock$FSReaderImpl.readAtOffset(HFileBlock.java:1564) at org.apache.hadoop.hbase.io.hfile.HFileBlock$FSReaderImpl.readBlockDataInternal(HFileBlock.java:1788) at org.apache.hadoop.hbase.io.hfile.HFileBlock$FSReaderImpl.readBlockData(HFileBlock.java:1597) at org.apache.hadoop.hbase.io.hfile.HFileReaderImpl.readBlock(HFileReaderImpl.java:1347) at org.apache.hadoop.hbase.io.hfile.HFileReaderImpl.readBlock(HFileReaderImpl.java:1268) at org.apache.hadoop.hbase.io.hfile.HFileReaderImpl$HFileScannerImpl.readNextDataBlock(HFileReaderImpl.java:762) at org.apache.hadoop.hbase.io.hfile.HFileReaderImpl$EncodedScanner.next(HFileReaderImpl.java:1540) at org.apache.hadoop.hbase.regionserver.StoreFileScanner.reseekAtOrAfter(StoreFileScanner.java:351) at org.apache.hadoop.hbase.regionserver.StoreFileScanner.reseek(StoreFileScanner.java:244) at org.apache.hadoop.hbase.regionserver.NonLazyKeyValueScanner.doRealSeek(NonLazyKeyValueScanner.java:55) at org.apache.hadoop.hbase.regionserver.KeyValueHeap.generalizedSeek(KeyValueHeap.java:322) at org.apache.hadoop.hbase.regionserver.KeyValueHeap.reseek(KeyValueHeap.java:265) at org.apache.hadoop.hbase.regionserver.StoreScanner.reseek(StoreScanner.java:1095) at org.apache.hadoop.hbase.regionserver.StoreScanner.seekAsDirection(StoreScanner.java:1084) at org.apache.hadoop.hbase.regionserver.StoreScanner.seekOrSkipToNextColumn(StoreScanner.java:821) at org.apache.hadoop.hbase.regionserver.StoreScanner.next(StoreScanner.java:684) at org.apache.hadoop.hbase.regionserver.KeyValueHeap.next(KeyValueHeap.java:155) at org.apache.hadoop.hbase.regionserver.HRegion$RegionScannerImpl.populateResult(HRegion.java:7399) at org.apache.hadoop.hbase.regionserver.HRegion$RegionScannerImpl.nextInternal(HRegion.java:7567) at org.apache.hadoop.hbase.regionserver.HRegion$RegionScannerImpl.nextRaw(HRegion.java:7331) at org.apache.hadoop.hbase.regionserver.HRegion$RegionScannerImpl.nextRaw(HRegion.java:7317) at org.apache.phoenix.coprocessor.PagedRegionScanner.next(PagedRegionScanner.java:57) at org.apache.phoenix.coprocessor.PagedRegionScanner.nextRaw(PagedRegionScanner.java:96) at org.apache.phoenix.iterate.RegionScannerFactory$1.nextRaw(RegionScannerFactory.java:211) at org.apache.phoenix.coprocessor.DelegateRegionScanner.nextRaw(DelegateRegionScanner.java:77) at org.apache.phoenix.coprocessor.DelegateRegionScanner.nextRaw(DelegateRegionScanner.java:77) at org.apache.phoenix.coprocessor.BaseScannerRegionObserver$RegionScannerHolder.nextRaw(BaseScannerRegionObserver.java:308) at org.apache.hadoop.hbase.regionserver.RSRpcServices.scan(RSRpcServices.java:3373) - locked <0x00007fc85f088fd8> (a org.apache.phoenix.coprocessor.BaseScannerRegionObserver$RegionScannerHolder) at org.apache.hadoop.hbase.regionserver.RSRpcServices.scan(RSRpcServices.java:3637) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45819) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:392) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:140) at org.apache.hadoop.hbase.ipc.RpcExecutor$Handler.run(RpcExecutor.java:359) at org.apache.hadoop.hbase.ipc.RpcExecutor$Handler.run(RpcExecutor.java:339) "RpcServer.default.FPBQ.Fifo.handler=35,queue=5,port=16020" #122 daemon prio=5 os_prio=0 tid=0x00007fc11d836000 nid=0x73ef waiting on condition [0x00007fc1157ea000] java.lang.Thread.State: WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007fc1c9febdb0> (a java.util.concurrent.Semaphore$NonfairSync) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer.parkAndCheckInterrupt(AbstractQueuedSynchronizer.java:836) at java.util.concurrent.locks.AbstractQueuedSynchronizer.doAcquireSharedInterruptibly(AbstractQueuedSynchronizer.java:997) at java.util.concurrent.locks.AbstractQueuedSynchronizer.acquireSharedInterruptibly(AbstractQueuedSynchronizer.java:1304) at java.util.concurrent.Semaphore.acquire(Semaphore.java:312) at org.apache.hadoop.hbase.ipc.FastPathBalancedQueueRpcExecutor$FastPathHandler.getCallRunner(FastPathBalancedQueueRpcExecutor.java:109) at org.apache.hadoop.hbase.ipc.RpcExecutor$Handler.run(RpcExecutor.java:339) "RpcServer.default.FPBQ.Fifo.handler=34,queue=4,port=16020" #121 daemon prio=5 os_prio=0 tid=0x00007fc11d834000 nid=0x73ee waiting for monitor entry [0x00007fc1158ea000] java.lang.Thread.State: BLOCKED (on object monitor) at org.apache.hadoop.fs.FSInputStream.read(FSInputStream.java:73) - waiting to lock <0x00007fc4c2769660> (a org.apache.hadoop.fs.s3a.S3AInputStream) at org.apache.hadoop.fs.FSDataInputStream.read(FSDataInputStream.java:97) at org.apache.hadoop.hbase.io.FileLink$FileLinkInputStream.read(FileLink.java:171) at org.apache.hadoop.fs.FSDataInputStream.read(FSDataInputStream.java:97) at org.apache.hadoop.hbase.io.util.BlockIOUtils.preadWithExtra(BlockIOUtils.java:235) at org.apache.hadoop.hbase.io.hfile.HFileBlock$FSReaderImpl.readAtOffset(HFileBlock.java:1564) at org.apache.hadoop.hbase.io.hfile.HFileBlock$FSReaderImpl.readBlockDataInternal(HFileBlock.java:1788) at org.apache.hadoop.hbase.io.hfile.HFileBlock$FSReaderImpl.readBlockData(HFileBlock.java:1597) at org.apache.hadoop.hbase.io.hfile.HFileReaderImpl.readBlock(HFileReaderImpl.java:1347) at org.apache.hadoop.hbase.io.hfile.HFileReaderImpl.readBlock(HFileReaderImpl.java:1268) at org.apache.hadoop.hbase.io.hfile.HFileReaderImpl$HFileScannerImpl.readNextDataBlock(HFileReaderImpl.java:762) at org.apache.hadoop.hbase.io.hfile.HFileReaderImpl$EncodedScanner.next(HFileReaderImpl.java:1540) at org.apache.hadoop.hbase.regionserver.StoreFileScanner.reseekAtOrAfter(StoreFileScanner.java:351) at org.apache.hadoop.hbase.regionserver.StoreFileScanner.reseek(StoreFileScanner.java:244) at org.apache.hadoop.hbase.regionserver.NonLazyKeyValueScanner.doRealSeek(NonLazyKeyValueScanner.java:55) at org.apache.hadoop.hbase.regionserver.KeyValueHeap.generalizedSeek(KeyValueHeap.java:322) at org.apache.hadoop.hbase.regionserver.KeyValueHeap.reseek(KeyValueHeap.java:265) at org.apache.hadoop.hbase.regionserver.StoreScanner.reseek(StoreScanner.java:1095) at org.apache.hadoop.hbase.regionserver.StoreScanner.seekAsDirection(StoreScanner.java:1084) at org.apache.hadoop.hbase.regionserver.StoreScanner.seekOrSkipToNextColumn(StoreScanner.java:821) at org.apache.hadoop.hbase.regionserver.StoreScanner.next(StoreScanner.java:684) at org.apache.hadoop.hbase.regionserver.KeyValueHeap.next(KeyValueHeap.java:155) at org.apache.hadoop.hbase.regionserver.HRegion$RegionScannerImpl.populateResult(HRegion.java:7399) at org.apache.hadoop.hbase.regionserver.HRegion$RegionScannerImpl.nextInternal(HRegion.java:7567) at org.apache.hadoop.hbase.regionserver.HRegion$RegionScannerImpl.nextRaw(HRegion.java:7331) at org.apache.hadoop.hbase.regionserver.HRegion$RegionScannerImpl.nextRaw(HRegion.java:7317) at org.apache.phoenix.coprocessor.PagedRegionScanner.next(PagedRegionScanner.java:57) at org.apache.phoenix.coprocessor.PagedRegionScanner.nextRaw(PagedRegionScanner.java:96) at org.apache.phoenix.iterate.RegionScannerFactory$1.nextRaw(RegionScannerFactory.java:211) at org.apache.phoenix.coprocessor.DelegateRegionScanner.nextRaw(DelegateRegionScanner.java:77) at org.apache.phoenix.coprocessor.DelegateRegionScanner.nextRaw(DelegateRegionScanner.java:77) at org.apache.phoenix.coprocessor.BaseScannerRegionObserver$RegionScannerHolder.nextRaw(BaseScannerRegionObserver.java:308) at org.apache.hadoop.hbase.regionserver.RSRpcServices.scan(RSRpcServices.java:3373) - locked <0x00007fc85f0948b0> (a org.apache.phoenix.coprocessor.BaseScannerRegionObserver$RegionScannerHolder) at org.apache.hadoop.hbase.regionserver.RSRpcServices.scan(RSRpcServices.java:3637) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45819) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:392) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:140) at org.apache.hadoop.hbase.ipc.RpcExecutor$Handler.run(RpcExecutor.java:359) at org.apache.hadoop.hbase.ipc.RpcExecutor$Handler.run(RpcExecutor.java:339) "RpcServer.default.FPBQ.Fifo.handler=33,queue=3,port=16020" #120 daemon prio=5 os_prio=0 tid=0x00007fc11d832000 nid=0x73ed waiting for monitor entry [0x00007fc1159eb000] java.lang.Thread.State: BLOCKED (on object monitor) at org.apache.hadoop.fs.FSInputStream.read(FSInputStream.java:73) - waiting to lock <0x00007fc4c2769660> (a org.apache.hadoop.fs.s3a.S3AInputStream) at org.apache.hadoop.fs.FSDataInputStream.read(FSDataInputStream.java:97) at org.apache.hadoop.hbase.io.FileLink$FileLinkInputStream.read(FileLink.java:171) at org.apache.hadoop.fs.FSDataInputStream.read(FSDataInputStream.java:97) at org.apache.hadoop.hbase.io.util.BlockIOUtils.preadWithExtra(BlockIOUtils.java:235) at org.apache.hadoop.hbase.io.hfile.HFileBlock$FSReaderImpl.readAtOffset(HFileBlock.java:1564) at org.apache.hadoop.hbase.io.hfile.HFileBlock$FSReaderImpl.readBlockDataInternal(HFileBlock.java:1788) at org.apache.hadoop.hbase.io.hfile.HFileBlock$FSReaderImpl.readBlockData(HFileBlock.java:1597) at org.apache.hadoop.hbase.io.hfile.HFileReaderImpl.readBlock(HFileReaderImpl.java:1347) at org.apache.hadoop.hbase.io.hfile.HFileReaderImpl.readBlock(HFileReaderImpl.java:1268) at org.apache.hadoop.hbase.io.hfile.HFileReaderImpl$HFileScannerImpl.readNextDataBlock(HFileReaderImpl.java:762) at org.apache.hadoop.hbase.io.hfile.HFileReaderImpl$EncodedScanner.next(HFileReaderImpl.java:1540) at org.apache.hadoop.hbase.regionserver.StoreFileScanner.reseekAtOrAfter(StoreFileScanner.java:351) at org.apache.hadoop.hbase.regionserver.StoreFileScanner.reseek(StoreFileScanner.java:244) at org.apache.hadoop.hbase.regionserver.NonLazyKeyValueScanner.doRealSeek(NonLazyKeyValueScanner.java:55) at org.apache.hadoop.hbase.regionserver.KeyValueHeap.generalizedSeek(KeyValueHeap.java:322) at org.apache.hadoop.hbase.regionserver.KeyValueHeap.reseek(KeyValueHeap.java:265) at org.apache.hadoop.hbase.regionserver.StoreScanner.reseek(StoreScanner.java:1095) at org.apache.hadoop.hbase.regionserver.StoreScanner.seekAsDirection(StoreScanner.java:1084) at org.apache.hadoop.hbase.regionserver.StoreScanner.seekOrSkipToNextColumn(StoreScanner.java:821) at org.apache.hadoop.hbase.regionserver.StoreScanner.next(StoreScanner.java:684) at org.apache.hadoop.hbase.regionserver.KeyValueHeap.next(KeyValueHeap.java:155) at org.apache.hadoop.hbase.regionserver.HRegion$RegionScannerImpl.populateResult(HRegion.java:7399) at org.apache.hadoop.hbase.regionserver.HRegion$RegionScannerImpl.nextInternal(HRegion.java:7567) at org.apache.hadoop.hbase.regionserver.HRegion$RegionScannerImpl.nextRaw(HRegion.java:7331) at org.apache.hadoop.hbase.regionserver.HRegion$RegionScannerImpl.nextRaw(HRegion.java:7317) at org.apache.phoenix.coprocessor.PagedRegionScanner.next(PagedRegionScanner.java:57) at org.apache.phoenix.coprocessor.PagedRegionScanner.nextRaw(PagedRegionScanner.java:96) at org.apache.phoenix.iterate.RegionScannerFactory$1.nextRaw(RegionScannerFactory.java:211) at org.apache.phoenix.coprocessor.DelegateRegionScanner.nextRaw(DelegateRegionScanner.java:77) at org.apache.phoenix.coprocessor.DelegateRegionScanner.nextRaw(DelegateRegionScanner.java:77) at org.apache.phoenix.coprocessor.BaseScannerRegionObserver$RegionScannerHolder.nextRaw(BaseScannerRegionObserver.java:308) at org.apache.hadoop.hbase.regionserver.RSRpcServices.scan(RSRpcServices.java:3373) - locked <0x00007fc85f08b108> (a org.apache.phoenix.coprocessor.BaseScannerRegionObserver$RegionScannerHolder) at org.apache.hadoop.hbase.regionserver.RSRpcServices.scan(RSRpcServices.java:3637) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45819) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:392) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:140) at org.apache.hadoop.hbase.ipc.RpcExecutor$Handler.run(RpcExecutor.java:359) at org.apache.hadoop.hbase.ipc.RpcExecutor$Handler.run(RpcExecutor.java:339) "RpcServer.default.FPBQ.Fifo.handler=32,queue=2,port=16020" #119 daemon prio=5 os_prio=0 tid=0x00007fc11d830000 nid=0x73ec waiting for monitor entry [0x00007fc115aec000] java.lang.Thread.State: BLOCKED (on object monitor) at org.apache.hadoop.fs.FSInputStream.read(FSInputStream.java:73) - waiting to lock <0x00007fc4c2769660> (a org.apache.hadoop.fs.s3a.S3AInputStream) at org.apache.hadoop.fs.FSDataInputStream.read(FSDataInputStream.java:97) at org.apache.hadoop.hbase.io.FileLink$FileLinkInputStream.read(FileLink.java:171) at org.apache.hadoop.fs.FSDataInputStream.read(FSDataInputStream.java:97) at org.apache.hadoop.hbase.io.util.BlockIOUtils.preadWithExtra(BlockIOUtils.java:235) at org.apache.hadoop.hbase.io.hfile.HFileBlock$FSReaderImpl.readAtOffset(HFileBlock.java:1564) at org.apache.hadoop.hbase.io.hfile.HFileBlock$FSReaderImpl.readBlockDataInternal(HFileBlock.java:1788) at org.apache.hadoop.hbase.io.hfile.HFileBlock$FSReaderImpl.readBlockData(HFileBlock.java:1597) at org.apache.hadoop.hbase.io.hfile.HFileReaderImpl.readBlock(HFileReaderImpl.java:1347) at org.apache.hadoop.hbase.io.hfile.HFileReaderImpl.readBlock(HFileReaderImpl.java:1268) at org.apache.hadoop.hbase.io.hfile.HFileReaderImpl$HFileScannerImpl.readNextDataBlock(HFileReaderImpl.java:762) at org.apache.hadoop.hbase.io.hfile.HFileReaderImpl$EncodedScanner.next(HFileReaderImpl.java:1540) at org.apache.hadoop.hbase.regionserver.StoreFileScanner.reseekAtOrAfter(StoreFileScanner.java:351) at org.apache.hadoop.hbase.regionserver.StoreFileScanner.reseek(StoreFileScanner.java:244) at org.apache.hadoop.hbase.regionserver.NonLazyKeyValueScanner.doRealSeek(NonLazyKeyValueScanner.java:55) at org.apache.hadoop.hbase.regionserver.KeyValueHeap.generalizedSeek(KeyValueHeap.java:322) at org.apache.hadoop.hbase.regionserver.KeyValueHeap.reseek(KeyValueHeap.java:265) at org.apache.hadoop.hbase.regionserver.StoreScanner.reseek(StoreScanner.java:1095) at org.apache.hadoop.hbase.regionserver.StoreScanner.seekAsDirection(StoreScanner.java:1084) at org.apache.hadoop.hbase.regionserver.StoreScanner.seekOrSkipToNextColumn(StoreScanner.java:821) at org.apache.hadoop.hbase.regionserver.StoreScanner.next(StoreScanner.java:684) at org.apache.hadoop.hbase.regionserver.KeyValueHeap.next(KeyValueHeap.java:155) at org.apache.hadoop.hbase.regionserver.HRegion$RegionScannerImpl.populateResult(HRegion.java:7399) at org.apache.hadoop.hbase.regionserver.HRegion$RegionScannerImpl.nextInternal(HRegion.java:7567) at org.apache.hadoop.hbase.regionserver.HRegion$RegionScannerImpl.nextRaw(HRegion.java:7331) at org.apache.hadoop.hbase.regionserver.HRegion$RegionScannerImpl.nextRaw(HRegion.java:7317) at org.apache.phoenix.coprocessor.PagedRegionScanner.next(PagedRegionScanner.java:57) at org.apache.phoenix.coprocessor.PagedRegionScanner.nextRaw(PagedRegionScanner.java:96) at org.apache.phoenix.iterate.RegionScannerFactory$1.nextRaw(RegionScannerFactory.java:211) at org.apache.phoenix.coprocessor.DelegateRegionScanner.nextRaw(DelegateRegionScanner.java:77) at org.apache.phoenix.coprocessor.DelegateRegionScanner.nextRaw(DelegateRegionScanner.java:77) at org.apache.phoenix.coprocessor.BaseScannerRegionObserver$RegionScannerHolder.nextRaw(BaseScannerRegionObserver.java:308) at org.apache.hadoop.hbase.regionserver.RSRpcServices.scan(RSRpcServices.java:3373) - locked <0x00007fc85f0af190> (a org.apache.phoenix.coprocessor.BaseScannerRegionObserver$RegionScannerHolder) at org.apache.hadoop.hbase.regionserver.RSRpcServices.scan(RSRpcServices.java:3637) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45819) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:392) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:140) at org.apache.hadoop.hbase.ipc.RpcExecutor$Handler.run(RpcExecutor.java:359) at org.apache.hadoop.hbase.ipc.RpcExecutor$Handler.run(RpcExecutor.java:339) "RpcServer.default.FPBQ.Fifo.handler=31,queue=1,port=16020" #118 daemon prio=5 os_prio=0 tid=0x00007fc11d82d800 nid=0x73eb waiting for monitor entry [0x00007fc115bed000] java.lang.Thread.State: BLOCKED (on object monitor) at org.apache.hadoop.fs.FSInputStream.read(FSInputStream.java:73) - waiting to lock <0x00007fc4c2769660> (a org.apache.hadoop.fs.s3a.S3AInputStream) at org.apache.hadoop.fs.FSDataInputStream.read(FSDataInputStream.java:97) at org.apache.hadoop.hbase.io.FileLink$FileLinkInputStream.read(FileLink.java:171) at org.apache.hadoop.fs.FSDataInputStream.read(FSDataInputStream.java:97) at org.apache.hadoop.hbase.io.util.BlockIOUtils.preadWithExtra(BlockIOUtils.java:235) at org.apache.hadoop.hbase.io.hfile.HFileBlock$FSReaderImpl.readAtOffset(HFileBlock.java:1564) at org.apache.hadoop.hbase.io.hfile.HFileBlock$FSReaderImpl.readBlockDataInternal(HFileBlock.java:1788) at org.apache.hadoop.hbase.io.hfile.HFileBlock$FSReaderImpl.readBlockData(HFileBlock.java:1597) at org.apache.hadoop.hbase.io.hfile.HFileReaderImpl.readBlock(HFileReaderImpl.java:1347) at org.apache.hadoop.hbase.io.hfile.HFileReaderImpl.readBlock(HFileReaderImpl.java:1268) at org.apache.hadoop.hbase.io.hfile.HFileReaderImpl$HFileScannerImpl.readNextDataBlock(HFileReaderImpl.java:762) at org.apache.hadoop.hbase.io.hfile.HFileReaderImpl$EncodedScanner.next(HFileReaderImpl.java:1540) at org.apache.hadoop.hbase.regionserver.StoreFileScanner.reseekAtOrAfter(StoreFileScanner.java:351) at org.apache.hadoop.hbase.regionserver.StoreFileScanner.reseek(StoreFileScanner.java:244) at org.apache.hadoop.hbase.regionserver.NonLazyKeyValueScanner.doRealSeek(NonLazyKeyValueScanner.java:55) at org.apache.hadoop.hbase.regionserver.KeyValueHeap.generalizedSeek(KeyValueHeap.java:322) at org.apache.hadoop.hbase.regionserver.KeyValueHeap.reseek(KeyValueHeap.java:265) at org.apache.hadoop.hbase.regionserver.StoreScanner.reseek(StoreScanner.java:1095) at org.apache.hadoop.hbase.regionserver.StoreScanner.seekAsDirection(StoreScanner.java:1084) at org.apache.hadoop.hbase.regionserver.StoreScanner.seekOrSkipToNextColumn(StoreScanner.java:821) at org.apache.hadoop.hbase.regionserver.StoreScanner.next(StoreScanner.java:684) at org.apache.hadoop.hbase.regionserver.KeyValueHeap.next(KeyValueHeap.java:155) at org.apache.hadoop.hbase.regionserver.HRegion$RegionScannerImpl.populateResult(HRegion.java:7399) at org.apache.hadoop.hbase.regionserver.HRegion$RegionScannerImpl.nextInternal(HRegion.java:7567) at org.apache.hadoop.hbase.regionserver.HRegion$RegionScannerImpl.nextRaw(HRegion.java:7331) at org.apache.hadoop.hbase.regionserver.HRegion$RegionScannerImpl.nextRaw(HRegion.java:7317) at org.apache.phoenix.coprocessor.PagedRegionScanner.next(PagedRegionScanner.java:57) at org.apache.phoenix.coprocessor.PagedRegionScanner.nextRaw(PagedRegionScanner.java:96) at org.apache.phoenix.iterate.RegionScannerFactory$1.nextRaw(RegionScannerFactory.java:211) at org.apache.phoenix.coprocessor.DelegateRegionScanner.nextRaw(DelegateRegionScanner.java:77) at org.apache.phoenix.coprocessor.DelegateRegionScanner.nextRaw(DelegateRegionScanner.java:77) at org.apache.phoenix.coprocessor.BaseScannerRegionObserver$RegionScannerHolder.nextRaw(BaseScannerRegionObserver.java:308) at org.apache.hadoop.hbase.regionserver.RSRpcServices.scan(RSRpcServices.java:3373) - locked <0x00007fc85f0a7810> (a org.apache.phoenix.coprocessor.BaseScannerRegionObserver$RegionScannerHolder) at org.apache.hadoop.hbase.regionserver.RSRpcServices.scan(RSRpcServices.java:3637) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45819) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:392) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:140) at org.apache.hadoop.hbase.ipc.RpcExecutor$Handler.run(RpcExecutor.java:359) at org.apache.hadoop.hbase.ipc.RpcExecutor$Handler.run(RpcExecutor.java:339) "RpcServer.default.FPBQ.Fifo.handler=30,queue=0,port=16020" #117 daemon prio=5 os_prio=0 tid=0x00007fc11d82b800 nid=0x73ea waiting for monitor entry [0x00007fc115cee000] java.lang.Thread.State: BLOCKED (on object monitor) at org.apache.hadoop.fs.FSInputStream.read(FSInputStream.java:73) - waiting to lock <0x00007fc4c2769660> (a org.apache.hadoop.fs.s3a.S3AInputStream) at org.apache.hadoop.fs.FSDataInputStream.read(FSDataInputStream.java:97) at org.apache.hadoop.hbase.io.FileLink$FileLinkInputStream.read(FileLink.java:171) at org.apache.hadoop.fs.FSDataInputStream.read(FSDataInputStream.java:97) at org.apache.hadoop.hbase.io.util.BlockIOUtils.preadWithExtra(BlockIOUtils.java:235) at org.apache.hadoop.hbase.io.hfile.HFileBlock$FSReaderImpl.readAtOffset(HFileBlock.java:1564) at org.apache.hadoop.hbase.io.hfile.HFileBlock$FSReaderImpl.readBlockDataInternal(HFileBlock.java:1788) at org.apache.hadoop.hbase.io.hfile.HFileBlock$FSReaderImpl.readBlockData(HFileBlock.java:1597) at org.apache.hadoop.hbase.io.hfile.HFileReaderImpl.readBlock(HFileReaderImpl.java:1347) at org.apache.hadoop.hbase.io.hfile.HFileReaderImpl.readBlock(HFileReaderImpl.java:1268) at org.apache.hadoop.hbase.io.hfile.HFileReaderImpl$HFileScannerImpl.readNextDataBlock(HFileReaderImpl.java:762) at org.apache.hadoop.hbase.io.hfile.HFileReaderImpl$EncodedScanner.next(HFileReaderImpl.java:1540) at org.apache.hadoop.hbase.regionserver.StoreFileScanner.reseekAtOrAfter(StoreFileScanner.java:351) at org.apache.hadoop.hbase.regionserver.StoreFileScanner.reseek(StoreFileScanner.java:244) at org.apache.hadoop.hbase.regionserver.NonLazyKeyValueScanner.doRealSeek(NonLazyKeyValueScanner.java:55) at org.apache.hadoop.hbase.regionserver.KeyValueHeap.generalizedSeek(KeyValueHeap.java:322) at org.apache.hadoop.hbase.regionserver.KeyValueHeap.reseek(KeyValueHeap.java:265) at org.apache.hadoop.hbase.regionserver.StoreScanner.reseek(StoreScanner.java:1095) at org.apache.hadoop.hbase.regionserver.StoreScanner.seekAsDirection(StoreScanner.java:1084) at org.apache.hadoop.hbase.regionserver.StoreScanner.seekOrSkipToNextColumn(StoreScanner.java:821) at org.apache.hadoop.hbase.regionserver.StoreScanner.next(StoreScanner.java:684) at org.apache.hadoop.hbase.regionserver.KeyValueHeap.next(KeyValueHeap.java:155) at org.apache.hadoop.hbase.regionserver.HRegion$RegionScannerImpl.populateResult(HRegion.java:7399) at org.apache.hadoop.hbase.regionserver.HRegion$RegionScannerImpl.nextInternal(HRegion.java:7567) at org.apache.hadoop.hbase.regionserver.HRegion$RegionScannerImpl.nextRaw(HRegion.java:7331) at org.apache.hadoop.hbase.regionserver.HRegion$RegionScannerImpl.nextRaw(HRegion.java:7317) at org.apache.phoenix.coprocessor.PagedRegionScanner.next(PagedRegionScanner.java:57) at org.apache.phoenix.coprocessor.PagedRegionScanner.nextRaw(PagedRegionScanner.java:96) at org.apache.phoenix.iterate.RegionScannerFactory$1.nextRaw(RegionScannerFactory.java:211) at org.apache.phoenix.coprocessor.DelegateRegionScanner.nextRaw(DelegateRegionScanner.java:77) at org.apache.phoenix.coprocessor.DelegateRegionScanner.nextRaw(DelegateRegionScanner.java:77) at org.apache.phoenix.coprocessor.BaseScannerRegionObserver$RegionScannerHolder.nextRaw(BaseScannerRegionObserver.java:308) at org.apache.hadoop.hbase.regionserver.RSRpcServices.scan(RSRpcServices.java:3373) - locked <0x00007fc6b13a0438> (a org.apache.phoenix.coprocessor.BaseScannerRegionObserver$RegionScannerHolder) at org.apache.hadoop.hbase.regionserver.RSRpcServices.scan(RSRpcServices.java:3637) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45819) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:392) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:140) at org.apache.hadoop.hbase.ipc.RpcExecutor$Handler.run(RpcExecutor.java:359) at org.apache.hadoop.hbase.ipc.RpcExecutor$Handler.run(RpcExecutor.java:339) "RpcServer.default.FPBQ.Fifo.handler=29,queue=9,port=16020" #116 daemon prio=5 os_prio=0 tid=0x00007fc11d829800 nid=0x73e9 waiting for monitor entry [0x00007fc115def000] java.lang.Thread.State: BLOCKED (on object monitor) at org.apache.hadoop.fs.FSInputStream.read(FSInputStream.java:73) - waiting to lock <0x00007fc4c2769660> (a org.apache.hadoop.fs.s3a.S3AInputStream) at org.apache.hadoop.fs.FSDataInputStream.read(FSDataInputStream.java:97) at org.apache.hadoop.hbase.io.FileLink$FileLinkInputStream.read(FileLink.java:171) at org.apache.hadoop.fs.FSDataInputStream.read(FSDataInputStream.java:97) at org.apache.hadoop.hbase.io.util.BlockIOUtils.preadWithExtra(BlockIOUtils.java:235) at org.apache.hadoop.hbase.io.hfile.HFileBlock$FSReaderImpl.readAtOffset(HFileBlock.java:1564) at org.apache.hadoop.hbase.io.hfile.HFileBlock$FSReaderImpl.readBlockDataInternal(HFileBlock.java:1788) at org.apache.hadoop.hbase.io.hfile.HFileBlock$FSReaderImpl.readBlockData(HFileBlock.java:1597) at org.apache.hadoop.hbase.io.hfile.HFileReaderImpl.readBlock(HFileReaderImpl.java:1347) at org.apache.hadoop.hbase.io.hfile.HFileReaderImpl.readBlock(HFileReaderImpl.java:1268) at org.apache.hadoop.hbase.io.hfile.HFileReaderImpl$HFileScannerImpl.readNextDataBlock(HFileReaderImpl.java:762) at org.apache.hadoop.hbase.io.hfile.HFileReaderImpl$EncodedScanner.next(HFileReaderImpl.java:1540) at org.apache.hadoop.hbase.regionserver.StoreFileScanner.reseekAtOrAfter(StoreFileScanner.java:351) at org.apache.hadoop.hbase.regionserver.StoreFileScanner.reseek(StoreFileScanner.java:244) at org.apache.hadoop.hbase.regionserver.NonLazyKeyValueScanner.doRealSeek(NonLazyKeyValueScanner.java:55) at org.apache.hadoop.hbase.regionserver.KeyValueHeap.generalizedSeek(KeyValueHeap.java:322) at org.apache.hadoop.hbase.regionserver.KeyValueHeap.reseek(KeyValueHeap.java:265) at org.apache.hadoop.hbase.regionserver.StoreScanner.reseek(StoreScanner.java:1095) at org.apache.hadoop.hbase.regionserver.StoreScanner.seekAsDirection(StoreScanner.java:1084) at org.apache.hadoop.hbase.regionserver.StoreScanner.seekOrSkipToNextColumn(StoreScanner.java:821) at org.apache.hadoop.hbase.regionserver.StoreScanner.next(StoreScanner.java:684) at org.apache.hadoop.hbase.regionserver.KeyValueHeap.next(KeyValueHeap.java:155) at org.apache.hadoop.hbase.regionserver.HRegion$RegionScannerImpl.populateResult(HRegion.java:7399) at org.apache.hadoop.hbase.regionserver.HRegion$RegionScannerImpl.nextInternal(HRegion.java:7567) at org.apache.hadoop.hbase.regionserver.HRegion$RegionScannerImpl.nextRaw(HRegion.java:7331) at org.apache.hadoop.hbase.regionserver.HRegion$RegionScannerImpl.nextRaw(HRegion.java:7317) at org.apache.phoenix.coprocessor.PagedRegionScanner.next(PagedRegionScanner.java:57) at org.apache.phoenix.coprocessor.PagedRegionScanner.nextRaw(PagedRegionScanner.java:96) at org.apache.phoenix.iterate.RegionScannerFactory$1.nextRaw(RegionScannerFactory.java:211) at org.apache.phoenix.coprocessor.DelegateRegionScanner.nextRaw(DelegateRegionScanner.java:77) at org.apache.phoenix.coprocessor.DelegateRegionScanner.nextRaw(DelegateRegionScanner.java:77) at org.apache.phoenix.coprocessor.BaseScannerRegionObserver$RegionScannerHolder.nextRaw(BaseScannerRegionObserver.java:308) at org.apache.hadoop.hbase.regionserver.RSRpcServices.scan(RSRpcServices.java:3373) - locked <0x00007fc85f08d100> (a org.apache.phoenix.coprocessor.BaseScannerRegionObserver$RegionScannerHolder) at org.apache.hadoop.hbase.regionserver.RSRpcServices.scan(RSRpcServices.java:3637) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45819) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:392) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:140) at org.apache.hadoop.hbase.ipc.RpcExecutor$Handler.run(RpcExecutor.java:359) at org.apache.hadoop.hbase.ipc.RpcExecutor$Handler.run(RpcExecutor.java:339) "RpcServer.default.FPBQ.Fifo.handler=28,queue=8,port=16020" #115 daemon prio=5 os_prio=0 tid=0x00007fc11d827800 nid=0x73e8 waiting for monitor entry [0x00007fc115ef0000] java.lang.Thread.State: BLOCKED (on object monitor) at org.apache.hadoop.fs.FSInputStream.read(FSInputStream.java:73) - waiting to lock <0x00007fc4c2769660> (a org.apache.hadoop.fs.s3a.S3AInputStream) at org.apache.hadoop.fs.FSDataInputStream.read(FSDataInputStream.java:97) at org.apache.hadoop.hbase.io.FileLink$FileLinkInputStream.read(FileLink.java:171) at org.apache.hadoop.fs.FSDataInputStream.read(FSDataInputStream.java:97) at org.apache.hadoop.hbase.io.util.BlockIOUtils.preadWithExtra(BlockIOUtils.java:235) at org.apache.hadoop.hbase.io.hfile.HFileBlock$FSReaderImpl.readAtOffset(HFileBlock.java:1564) at org.apache.hadoop.hbase.io.hfile.HFileBlock$FSReaderImpl.readBlockDataInternal(HFileBlock.java:1788) at org.apache.hadoop.hbase.io.hfile.HFileBlock$FSReaderImpl.readBlockData(HFileBlock.java:1597) at org.apache.hadoop.hbase.io.hfile.HFileReaderImpl.readBlock(HFileReaderImpl.java:1347) at org.apache.hadoop.hbase.io.hfile.HFileReaderImpl.readBlock(HFileReaderImpl.java:1268) at org.apache.hadoop.hbase.io.hfile.HFileReaderImpl$HFileScannerImpl.readNextDataBlock(HFileReaderImpl.java:762) at org.apache.hadoop.hbase.io.hfile.HFileReaderImpl$EncodedScanner.next(HFileReaderImpl.java:1540) at org.apache.hadoop.hbase.regionserver.StoreFileScanner.reseekAtOrAfter(StoreFileScanner.java:351) at org.apache.hadoop.hbase.regionserver.StoreFileScanner.reseek(StoreFileScanner.java:244) at org.apache.hadoop.hbase.regionserver.NonLazyKeyValueScanner.doRealSeek(NonLazyKeyValueScanner.java:55) at org.apache.hadoop.hbase.regionserver.KeyValueHeap.generalizedSeek(KeyValueHeap.java:322) at org.apache.hadoop.hbase.regionserver.KeyValueHeap.reseek(KeyValueHeap.java:265) at org.apache.hadoop.hbase.regionserver.StoreScanner.reseek(StoreScanner.java:1095) at org.apache.hadoop.hbase.regionserver.StoreScanner.seekAsDirection(StoreScanner.java:1084) at org.apache.hadoop.hbase.regionserver.StoreScanner.seekOrSkipToNextColumn(StoreScanner.java:821) at org.apache.hadoop.hbase.regionserver.StoreScanner.next(StoreScanner.java:684) at org.apache.hadoop.hbase.regionserver.KeyValueHeap.next(KeyValueHeap.java:155) at org.apache.hadoop.hbase.regionserver.HRegion$RegionScannerImpl.populateResult(HRegion.java:7399) at org.apache.hadoop.hbase.regionserver.HRegion$RegionScannerImpl.nextInternal(HRegion.java:7567) at org.apache.hadoop.hbase.regionserver.HRegion$RegionScannerImpl.nextRaw(HRegion.java:7331) at org.apache.hadoop.hbase.regionserver.HRegion$RegionScannerImpl.nextRaw(HRegion.java:7317) at org.apache.phoenix.coprocessor.PagedRegionScanner.next(PagedRegionScanner.java:57) at org.apache.phoenix.coprocessor.PagedRegionScanner.nextRaw(PagedRegionScanner.java:96) at org.apache.phoenix.iterate.RegionScannerFactory$1.nextRaw(RegionScannerFactory.java:211) at org.apache.phoenix.coprocessor.DelegateRegionScanner.nextRaw(DelegateRegionScanner.java:77) at org.apache.phoenix.coprocessor.DelegateRegionScanner.nextRaw(DelegateRegionScanner.java:77) at org.apache.phoenix.coprocessor.BaseScannerRegionObserver$RegionScannerHolder.nextRaw(BaseScannerRegionObserver.java:308) at org.apache.hadoop.hbase.regionserver.RSRpcServices.scan(RSRpcServices.java:3373) - locked <0x00007fc6b1396878> (a org.apache.phoenix.coprocessor.BaseScannerRegionObserver$RegionScannerHolder) at org.apache.hadoop.hbase.regionserver.RSRpcServices.scan(RSRpcServices.java:3637) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45819) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:392) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:140) at org.apache.hadoop.hbase.ipc.RpcExecutor$Handler.run(RpcExecutor.java:359) at org.apache.hadoop.hbase.ipc.RpcExecutor$Handler.run(RpcExecutor.java:339) "RpcServer.default.FPBQ.Fifo.handler=27,queue=7,port=16020" #114 daemon prio=5 os_prio=0 tid=0x00007fc11d825800 nid=0x73e7 waiting for monitor entry [0x00007fc115ff1000] java.lang.Thread.State: BLOCKED (on object monitor) at org.apache.hadoop.fs.FSInputStream.read(FSInputStream.java:73) - waiting to lock <0x00007fc4c2769660> (a org.apache.hadoop.fs.s3a.S3AInputStream) at org.apache.hadoop.fs.FSDataInputStream.read(FSDataInputStream.java:97) at org.apache.hadoop.hbase.io.FileLink$FileLinkInputStream.read(FileLink.java:171) at org.apache.hadoop.fs.FSDataInputStream.read(FSDataInputStream.java:97) at org.apache.hadoop.hbase.io.util.BlockIOUtils.preadWithExtra(BlockIOUtils.java:235) at org.apache.hadoop.hbase.io.hfile.HFileBlock$FSReaderImpl.readAtOffset(HFileBlock.java:1564) at org.apache.hadoop.hbase.io.hfile.HFileBlock$FSReaderImpl.readBlockDataInternal(HFileBlock.java:1788) at org.apache.hadoop.hbase.io.hfile.HFileBlock$FSReaderImpl.readBlockData(HFileBlock.java:1597) at org.apache.hadoop.hbase.io.hfile.HFileReaderImpl.readBlock(HFileReaderImpl.java:1347) at org.apache.hadoop.hbase.io.hfile.HFileReaderImpl.readBlock(HFileReaderImpl.java:1268) at org.apache.hadoop.hbase.io.hfile.HFileReaderImpl$HFileScannerImpl.readNextDataBlock(HFileReaderImpl.java:762) at org.apache.hadoop.hbase.io.hfile.HFileReaderImpl$EncodedScanner.next(HFileReaderImpl.java:1540) at org.apache.hadoop.hbase.regionserver.StoreFileScanner.reseekAtOrAfter(StoreFileScanner.java:351) at org.apache.hadoop.hbase.regionserver.StoreFileScanner.reseek(StoreFileScanner.java:244) at org.apache.hadoop.hbase.regionserver.NonLazyKeyValueScanner.doRealSeek(NonLazyKeyValueScanner.java:55) at org.apache.hadoop.hbase.regionserver.KeyValueHeap.generalizedSeek(KeyValueHeap.java:322) at org.apache.hadoop.hbase.regionserver.KeyValueHeap.reseek(KeyValueHeap.java:265) at org.apache.hadoop.hbase.regionserver.StoreScanner.reseek(StoreScanner.java:1095) at org.apache.hadoop.hbase.regionserver.StoreScanner.seekAsDirection(StoreScanner.java:1084) at org.apache.hadoop.hbase.regionserver.StoreScanner.seekOrSkipToNextColumn(StoreScanner.java:821) at org.apache.hadoop.hbase.regionserver.StoreScanner.next(StoreScanner.java:684) at org.apache.hadoop.hbase.regionserver.KeyValueHeap.next(KeyValueHeap.java:155) at org.apache.hadoop.hbase.regionserver.HRegion$RegionScannerImpl.populateResult(HRegion.java:7399) at org.apache.hadoop.hbase.regionserver.HRegion$RegionScannerImpl.nextInternal(HRegion.java:7567) at org.apache.hadoop.hbase.regionserver.HRegion$RegionScannerImpl.nextRaw(HRegion.java:7331) at org.apache.hadoop.hbase.regionserver.HRegion$RegionScannerImpl.nextRaw(HRegion.java:7317) at org.apache.phoenix.coprocessor.PagedRegionScanner.next(PagedRegionScanner.java:57) at org.apache.phoenix.coprocessor.PagedRegionScanner.nextRaw(PagedRegionScanner.java:96) at org.apache.phoenix.iterate.RegionScannerFactory$1.nextRaw(RegionScannerFactory.java:211) at org.apache.phoenix.coprocessor.DelegateRegionScanner.nextRaw(DelegateRegionScanner.java:77) at org.apache.phoenix.coprocessor.DelegateRegionScanner.nextRaw(DelegateRegionScanner.java:77) at org.apache.phoenix.coprocessor.BaseScannerRegionObserver$RegionScannerHolder.nextRaw(BaseScannerRegionObserver.java:308) at org.apache.hadoop.hbase.regionserver.RSRpcServices.scan(RSRpcServices.java:3373) - locked <0x00007fc85f081b58> (a org.apache.phoenix.coprocessor.BaseScannerRegionObserver$RegionScannerHolder) at org.apache.hadoop.hbase.regionserver.RSRpcServices.scan(RSRpcServices.java:3637) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45819) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:392) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:140) at org.apache.hadoop.hbase.ipc.RpcExecutor$Handler.run(RpcExecutor.java:359) at org.apache.hadoop.hbase.ipc.RpcExecutor$Handler.run(RpcExecutor.java:339) "RpcServer.default.FPBQ.Fifo.handler=26,queue=6,port=16020" #113 daemon prio=5 os_prio=0 tid=0x00007fc11d823800 nid=0x73e6 runnable [0x00007fc1160f2000] java.lang.Thread.State: RUNNABLE at org.apache.hadoop.hbase.io.encoding.FastDiffDeltaEncoder$FastDiffSeekerStateBufferedEncodedSeeker.decode(FastDiffDeltaEncoder.java:418) at org.apache.hadoop.hbase.io.encoding.FastDiffDeltaEncoder$FastDiffSeekerStateBufferedEncodedSeeker.decodeNext(FastDiffDeltaEncoder.java:525) at org.apache.hadoop.hbase.io.encoding.BufferedDataBlockEncoder$BufferedEncodedSeeker.seekToKeyInBlock(BufferedDataBlockEncoder.java:926) at org.apache.hadoop.hbase.io.hfile.HFileReaderImpl$EncodedScanner.loadBlockAndSeekToKey(HFileReaderImpl.java:1602) at org.apache.hadoop.hbase.io.hfile.HFileReaderImpl$HFileScannerImpl.seekTo(HFileReaderImpl.java:686) at org.apache.hadoop.hbase.io.hfile.HFileReaderImpl$HFileScannerImpl.reseekTo(HFileReaderImpl.java:662) at org.apache.hadoop.hbase.regionserver.StoreFileScanner.reseekAtOrAfter(StoreFileScanner.java:335) at org.apache.hadoop.hbase.regionserver.StoreFileScanner.reseek(StoreFileScanner.java:244) at org.apache.hadoop.hbase.regionserver.StoreFileScanner.enforceSeek(StoreFileScanner.java:457) at org.apache.hadoop.hbase.regionserver.KeyValueHeap.pollRealKV(KeyValueHeap.java:371) at org.apache.hadoop.hbase.regionserver.KeyValueHeap.generalizedSeek(KeyValueHeap.java:313) at org.apache.hadoop.hbase.regionserver.KeyValueHeap.requestSeek(KeyValueHeap.java:277) at org.apache.hadoop.hbase.regionserver.StoreScanner.reseek(StoreScanner.java:1093) at org.apache.hadoop.hbase.regionserver.StoreScanner.seekAsDirection(StoreScanner.java:1084) at org.apache.hadoop.hbase.regionserver.StoreScanner.next(StoreScanner.java:745) at org.apache.hadoop.hbase.regionserver.KeyValueHeap.next(KeyValueHeap.java:155) at org.apache.hadoop.hbase.regionserver.HRegion$RegionScannerImpl.populateResult(HRegion.java:7399) at org.apache.hadoop.hbase.regionserver.HRegion$RegionScannerImpl.nextInternal(HRegion.java:7567) at org.apache.hadoop.hbase.regionserver.HRegion$RegionScannerImpl.nextRaw(HRegion.java:7331) at org.apache.hadoop.hbase.regionserver.HRegion$RegionScannerImpl.nextRaw(HRegion.java:7317) at org.apache.phoenix.coprocessor.PagedRegionScanner.next(PagedRegionScanner.java:57) at org.apache.phoenix.coprocessor.PagedRegionScanner.nextRaw(PagedRegionScanner.java:96) at org.apache.phoenix.iterate.RegionScannerFactory$1.nextRaw(RegionScannerFactory.java:211) at org.apache.phoenix.coprocessor.DelegateRegionScanner.nextRaw(DelegateRegionScanner.java:77) at org.apache.phoenix.coprocessor.DelegateRegionScanner.nextRaw(DelegateRegionScanner.java:77) at org.apache.phoenix.coprocessor.BaseScannerRegionObserver$RegionScannerHolder.nextRaw(BaseScannerRegionObserver.java:308) at org.apache.hadoop.hbase.regionserver.RSRpcServices.scan(RSRpcServices.java:3373) - locked <0x00007fc911033d00> (a org.apache.phoenix.coprocessor.BaseScannerRegionObserver$RegionScannerHolder) at org.apache.hadoop.hbase.regionserver.RSRpcServices.scan(RSRpcServices.java:3637) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45819) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:392) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:140) at org.apache.hadoop.hbase.ipc.RpcExecutor$Handler.run(RpcExecutor.java:359) at org.apache.hadoop.hbase.ipc.RpcExecutor$Handler.run(RpcExecutor.java:339) "RpcServer.default.FPBQ.Fifo.handler=25,queue=5,port=16020" #112 daemon prio=5 os_prio=0 tid=0x00007fc11d821000 nid=0x73e5 waiting for monitor entry [0x00007fc1161f3000] java.lang.Thread.State: BLOCKED (on object monitor) at org.apache.hadoop.fs.FSInputStream.read(FSInputStream.java:73) - waiting to lock <0x00007fc4c2769660> (a org.apache.hadoop.fs.s3a.S3AInputStream) at org.apache.hadoop.fs.FSDataInputStream.read(FSDataInputStream.java:97) at org.apache.hadoop.hbase.io.FileLink$FileLinkInputStream.read(FileLink.java:171) at org.apache.hadoop.fs.FSDataInputStream.read(FSDataInputStream.java:97) at org.apache.hadoop.hbase.io.util.BlockIOUtils.preadWithExtra(BlockIOUtils.java:235) at org.apache.hadoop.hbase.io.hfile.HFileBlock$FSReaderImpl.readAtOffset(HFileBlock.java:1564) at org.apache.hadoop.hbase.io.hfile.HFileBlock$FSReaderImpl.readBlockDataInternal(HFileBlock.java:1788) at org.apache.hadoop.hbase.io.hfile.HFileBlock$FSReaderImpl.readBlockData(HFileBlock.java:1597) at org.apache.hadoop.hbase.io.hfile.HFileReaderImpl.readBlock(HFileReaderImpl.java:1347) at org.apache.hadoop.hbase.io.hfile.HFileReaderImpl.readBlock(HFileReaderImpl.java:1268) at org.apache.hadoop.hbase.io.hfile.HFileReaderImpl$HFileScannerImpl.readNextDataBlock(HFileReaderImpl.java:762) at org.apache.hadoop.hbase.io.hfile.HFileReaderImpl$EncodedScanner.next(HFileReaderImpl.java:1540) at org.apache.hadoop.hbase.regionserver.StoreFileScanner.reseekAtOrAfter(StoreFileScanner.java:351) at org.apache.hadoop.hbase.regionserver.StoreFileScanner.reseek(StoreFileScanner.java:244) at org.apache.hadoop.hbase.regionserver.NonLazyKeyValueScanner.doRealSeek(NonLazyKeyValueScanner.java:55) at org.apache.hadoop.hbase.regionserver.KeyValueHeap.generalizedSeek(KeyValueHeap.java:322) at org.apache.hadoop.hbase.regionserver.KeyValueHeap.reseek(KeyValueHeap.java:265) at org.apache.hadoop.hbase.regionserver.StoreScanner.reseek(StoreScanner.java:1095) at org.apache.hadoop.hbase.regionserver.StoreScanner.seekAsDirection(StoreScanner.java:1084) at org.apache.hadoop.hbase.regionserver.StoreScanner.seekOrSkipToNextColumn(StoreScanner.java:821) at org.apache.hadoop.hbase.regionserver.StoreScanner.next(StoreScanner.java:684) at org.apache.hadoop.hbase.regionserver.KeyValueHeap.next(KeyValueHeap.java:155) at org.apache.hadoop.hbase.regionserver.HRegion$RegionScannerImpl.populateResult(HRegion.java:7399) at org.apache.hadoop.hbase.regionserver.HRegion$RegionScannerImpl.nextInternal(HRegion.java:7567) at org.apache.hadoop.hbase.regionserver.HRegion$RegionScannerImpl.nextRaw(HRegion.java:7331) at org.apache.hadoop.hbase.regionserver.HRegion$RegionScannerImpl.nextRaw(HRegion.java:7317) at org.apache.phoenix.coprocessor.PagedRegionScanner.next(PagedRegionScanner.java:57) at org.apache.phoenix.coprocessor.PagedRegionScanner.nextRaw(PagedRegionScanner.java:96) at org.apache.phoenix.iterate.RegionScannerFactory$1.nextRaw(RegionScannerFactory.java:211) at org.apache.phoenix.coprocessor.DelegateRegionScanner.nextRaw(DelegateRegionScanner.java:77) at org.apache.phoenix.coprocessor.DelegateRegionScanner.nextRaw(DelegateRegionScanner.java:77) at org.apache.phoenix.coprocessor.BaseScannerRegionObserver$RegionScannerHolder.nextRaw(BaseScannerRegionObserver.java:308) at org.apache.hadoop.hbase.regionserver.RSRpcServices.scan(RSRpcServices.java:3373) - locked <0x00007fc85f098618> (a org.apache.phoenix.coprocessor.BaseScannerRegionObserver$RegionScannerHolder) at org.apache.hadoop.hbase.regionserver.RSRpcServices.scan(RSRpcServices.java:3637) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45819) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:392) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:140) at org.apache.hadoop.hbase.ipc.RpcExecutor$Handler.run(RpcExecutor.java:359) at org.apache.hadoop.hbase.ipc.RpcExecutor$Handler.run(RpcExecutor.java:339) "RpcServer.default.FPBQ.Fifo.handler=24,queue=4,port=16020" #111 daemon prio=5 os_prio=0 tid=0x00007fc11d81f800 nid=0x73e4 waiting on condition [0x00007fc1162f5000] java.lang.Thread.State: WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007fc1c9ff4a40> (a java.util.concurrent.Semaphore$NonfairSync) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer.parkAndCheckInterrupt(AbstractQueuedSynchronizer.java:836) at java.util.concurrent.locks.AbstractQueuedSynchronizer.doAcquireSharedInterruptibly(AbstractQueuedSynchronizer.java:997) at java.util.concurrent.locks.AbstractQueuedSynchronizer.acquireSharedInterruptibly(AbstractQueuedSynchronizer.java:1304) at java.util.concurrent.Semaphore.acquire(Semaphore.java:312) at org.apache.hadoop.hbase.ipc.FastPathBalancedQueueRpcExecutor$FastPathHandler.getCallRunner(FastPathBalancedQueueRpcExecutor.java:109) at org.apache.hadoop.hbase.ipc.RpcExecutor$Handler.run(RpcExecutor.java:339) "RpcServer.default.FPBQ.Fifo.handler=23,queue=3,port=16020" #110 daemon prio=5 os_prio=0 tid=0x00007fc11d81d000 nid=0x73e3 waiting for monitor entry [0x00007fc1163f5000] java.lang.Thread.State: BLOCKED (on object monitor) at org.apache.hadoop.fs.FSInputStream.read(FSInputStream.java:73) - waiting to lock <0x00007fc4c2769660> (a org.apache.hadoop.fs.s3a.S3AInputStream) at org.apache.hadoop.fs.FSDataInputStream.read(FSDataInputStream.java:97) at org.apache.hadoop.hbase.io.FileLink$FileLinkInputStream.read(FileLink.java:171) at org.apache.hadoop.fs.FSDataInputStream.read(FSDataInputStream.java:97) at org.apache.hadoop.hbase.io.util.BlockIOUtils.preadWithExtra(BlockIOUtils.java:235) at org.apache.hadoop.hbase.io.hfile.HFileBlock$FSReaderImpl.readAtOffset(HFileBlock.java:1564) at org.apache.hadoop.hbase.io.hfile.HFileBlock$FSReaderImpl.readBlockDataInternal(HFileBlock.java:1788) at org.apache.hadoop.hbase.io.hfile.HFileBlock$FSReaderImpl.readBlockData(HFileBlock.java:1597) at org.apache.hadoop.hbase.io.hfile.HFileReaderImpl.readBlock(HFileReaderImpl.java:1347) at org.apache.hadoop.hbase.io.hfile.HFileReaderImpl.readBlock(HFileReaderImpl.java:1268) at org.apache.hadoop.hbase.io.hfile.HFileReaderImpl$HFileScannerImpl.readNextDataBlock(HFileReaderImpl.java:762) at org.apache.hadoop.hbase.io.hfile.HFileReaderImpl$EncodedScanner.next(HFileReaderImpl.java:1540) at org.apache.hadoop.hbase.regionserver.StoreFileScanner.reseekAtOrAfter(StoreFileScanner.java:351) at org.apache.hadoop.hbase.regionserver.StoreFileScanner.reseek(StoreFileScanner.java:244) at org.apache.hadoop.hbase.regionserver.NonLazyKeyValueScanner.doRealSeek(NonLazyKeyValueScanner.java:55) at org.apache.hadoop.hbase.regionserver.KeyValueHeap.generalizedSeek(KeyValueHeap.java:322) at org.apache.hadoop.hbase.regionserver.KeyValueHeap.reseek(KeyValueHeap.java:265) at org.apache.hadoop.hbase.regionserver.StoreScanner.reseek(StoreScanner.java:1095) at org.apache.hadoop.hbase.regionserver.StoreScanner.seekAsDirection(StoreScanner.java:1084) at org.apache.hadoop.hbase.regionserver.StoreScanner.seekOrSkipToNextColumn(StoreScanner.java:821) at org.apache.hadoop.hbase.regionserver.StoreScanner.next(StoreScanner.java:684) at org.apache.hadoop.hbase.regionserver.KeyValueHeap.next(KeyValueHeap.java:155) at org.apache.hadoop.hbase.regionserver.HRegion$RegionScannerImpl.populateResult(HRegion.java:7399) at org.apache.hadoop.hbase.regionserver.HRegion$RegionScannerImpl.nextInternal(HRegion.java:7567) at org.apache.hadoop.hbase.regionserver.HRegion$RegionScannerImpl.nextRaw(HRegion.java:7331) at org.apache.hadoop.hbase.regionserver.HRegion$RegionScannerImpl.nextRaw(HRegion.java:7317) at org.apache.phoenix.coprocessor.PagedRegionScanner.next(PagedRegionScanner.java:57) at org.apache.phoenix.coprocessor.PagedRegionScanner.nextRaw(PagedRegionScanner.java:96) at org.apache.phoenix.iterate.RegionScannerFactory$1.nextRaw(RegionScannerFactory.java:211) at org.apache.phoenix.coprocessor.DelegateRegionScanner.nextRaw(DelegateRegionScanner.java:77) at org.apache.phoenix.coprocessor.DelegateRegionScanner.nextRaw(DelegateRegionScanner.java:77) at org.apache.phoenix.coprocessor.BaseScannerRegionObserver$RegionScannerHolder.nextRaw(BaseScannerRegionObserver.java:308) at org.apache.hadoop.hbase.regionserver.RSRpcServices.scan(RSRpcServices.java:3373) - locked <0x00007fc6b139cb70> (a org.apache.phoenix.coprocessor.BaseScannerRegionObserver$RegionScannerHolder) at org.apache.hadoop.hbase.regionserver.RSRpcServices.scan(RSRpcServices.java:3637) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45819) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:392) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:140) at org.apache.hadoop.hbase.ipc.RpcExecutor$Handler.run(RpcExecutor.java:359) at org.apache.hadoop.hbase.ipc.RpcExecutor$Handler.run(RpcExecutor.java:339) "RpcServer.default.FPBQ.Fifo.handler=22,queue=2,port=16020" #109 daemon prio=5 os_prio=0 tid=0x00007fc11d81b000 nid=0x73e2 waiting for monitor entry [0x00007fc1164f6000] java.lang.Thread.State: BLOCKED (on object monitor) at org.apache.hadoop.fs.FSInputStream.read(FSInputStream.java:73) - waiting to lock <0x00007fc4c2769660> (a org.apache.hadoop.fs.s3a.S3AInputStream) at org.apache.hadoop.fs.FSDataInputStream.read(FSDataInputStream.java:97) at org.apache.hadoop.hbase.io.FileLink$FileLinkInputStream.read(FileLink.java:171) at org.apache.hadoop.fs.FSDataInputStream.read(FSDataInputStream.java:97) at org.apache.hadoop.hbase.io.util.BlockIOUtils.preadWithExtra(BlockIOUtils.java:235) at org.apache.hadoop.hbase.io.hfile.HFileBlock$FSReaderImpl.readAtOffset(HFileBlock.java:1564) at org.apache.hadoop.hbase.io.hfile.HFileBlock$FSReaderImpl.readBlockDataInternal(HFileBlock.java:1788) at org.apache.hadoop.hbase.io.hfile.HFileBlock$FSReaderImpl.readBlockData(HFileBlock.java:1597) at org.apache.hadoop.hbase.io.hfile.HFileReaderImpl.readBlock(HFileReaderImpl.java:1347) at org.apache.hadoop.hbase.io.hfile.HFileReaderImpl.readBlock(HFileReaderImpl.java:1268) at org.apache.hadoop.hbase.io.hfile.HFileReaderImpl$HFileScannerImpl.readNextDataBlock(HFileReaderImpl.java:762) at org.apache.hadoop.hbase.io.hfile.HFileReaderImpl$EncodedScanner.next(HFileReaderImpl.java:1540) at org.apache.hadoop.hbase.regionserver.StoreFileScanner.reseekAtOrAfter(StoreFileScanner.java:351) at org.apache.hadoop.hbase.regionserver.StoreFileScanner.reseek(StoreFileScanner.java:244) at org.apache.hadoop.hbase.regionserver.NonLazyKeyValueScanner.doRealSeek(NonLazyKeyValueScanner.java:55) at org.apache.hadoop.hbase.regionserver.KeyValueHeap.generalizedSeek(KeyValueHeap.java:322) at org.apache.hadoop.hbase.regionserver.KeyValueHeap.reseek(KeyValueHeap.java:265) at org.apache.hadoop.hbase.regionserver.StoreScanner.reseek(StoreScanner.java:1095) at org.apache.hadoop.hbase.regionserver.StoreScanner.seekAsDirection(StoreScanner.java:1084) at org.apache.hadoop.hbase.regionserver.StoreScanner.seekOrSkipToNextColumn(StoreScanner.java:821) at org.apache.hadoop.hbase.regionserver.StoreScanner.next(StoreScanner.java:684) at org.apache.hadoop.hbase.regionserver.KeyValueHeap.next(KeyValueHeap.java:155) at org.apache.hadoop.hbase.regionserver.HRegion$RegionScannerImpl.populateResult(HRegion.java:7399) at org.apache.hadoop.hbase.regionserver.HRegion$RegionScannerImpl.nextInternal(HRegion.java:7567) at org.apache.hadoop.hbase.regionserver.HRegion$RegionScannerImpl.nextRaw(HRegion.java:7331) at org.apache.hadoop.hbase.regionserver.HRegion$RegionScannerImpl.nextRaw(HRegion.java:7317) at org.apache.phoenix.coprocessor.PagedRegionScanner.next(PagedRegionScanner.java:57) at org.apache.phoenix.coprocessor.PagedRegionScanner.nextRaw(PagedRegionScanner.java:96) at org.apache.phoenix.iterate.RegionScannerFactory$1.nextRaw(RegionScannerFactory.java:211) at org.apache.phoenix.coprocessor.DelegateRegionScanner.nextRaw(DelegateRegionScanner.java:77) at org.apache.phoenix.coprocessor.DelegateRegionScanner.nextRaw(DelegateRegionScanner.java:77) at org.apache.phoenix.coprocessor.BaseScannerRegionObserver$RegionScannerHolder.nextRaw(BaseScannerRegionObserver.java:308) at org.apache.hadoop.hbase.regionserver.RSRpcServices.scan(RSRpcServices.java:3373) - locked <0x00007fc6b13a0d60> (a org.apache.phoenix.coprocessor.BaseScannerRegionObserver$RegionScannerHolder) at org.apache.hadoop.hbase.regionserver.RSRpcServices.scan(RSRpcServices.java:3637) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45819) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:392) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:140) at org.apache.hadoop.hbase.ipc.RpcExecutor$Handler.run(RpcExecutor.java:359) at org.apache.hadoop.hbase.ipc.RpcExecutor$Handler.run(RpcExecutor.java:339) "RpcServer.default.FPBQ.Fifo.handler=21,queue=1,port=16020" #108 daemon prio=5 os_prio=0 tid=0x00007fc11d819000 nid=0x73e1 waiting for monitor entry [0x00007fc1165f7000] java.lang.Thread.State: BLOCKED (on object monitor) at org.apache.hadoop.fs.FSInputStream.read(FSInputStream.java:73) - waiting to lock <0x00007fc4c2769660> (a org.apache.hadoop.fs.s3a.S3AInputStream) at org.apache.hadoop.fs.FSDataInputStream.read(FSDataInputStream.java:97) at org.apache.hadoop.hbase.io.FileLink$FileLinkInputStream.read(FileLink.java:171) at org.apache.hadoop.fs.FSDataInputStream.read(FSDataInputStream.java:97) at org.apache.hadoop.hbase.io.util.BlockIOUtils.preadWithExtra(BlockIOUtils.java:235) at org.apache.hadoop.hbase.io.hfile.HFileBlock$FSReaderImpl.readAtOffset(HFileBlock.java:1564) at org.apache.hadoop.hbase.io.hfile.HFileBlock$FSReaderImpl.readBlockDataInternal(HFileBlock.java:1788) at org.apache.hadoop.hbase.io.hfile.HFileBlock$FSReaderImpl.readBlockData(HFileBlock.java:1597) at org.apache.hadoop.hbase.io.hfile.HFileReaderImpl.readBlock(HFileReaderImpl.java:1347) at org.apache.hadoop.hbase.io.hfile.HFileReaderImpl.readBlock(HFileReaderImpl.java:1268) at org.apache.hadoop.hbase.io.hfile.HFileReaderImpl$HFileScannerImpl.readNextDataBlock(HFileReaderImpl.java:762) at org.apache.hadoop.hbase.io.hfile.HFileReaderImpl$EncodedScanner.next(HFileReaderImpl.java:1540) at org.apache.hadoop.hbase.regionserver.StoreFileScanner.reseekAtOrAfter(StoreFileScanner.java:351) at org.apache.hadoop.hbase.regionserver.StoreFileScanner.reseek(StoreFileScanner.java:244) at org.apache.hadoop.hbase.regionserver.NonLazyKeyValueScanner.doRealSeek(NonLazyKeyValueScanner.java:55) at org.apache.hadoop.hbase.regionserver.KeyValueHeap.generalizedSeek(KeyValueHeap.java:322) at org.apache.hadoop.hbase.regionserver.KeyValueHeap.reseek(KeyValueHeap.java:265) at org.apache.hadoop.hbase.regionserver.StoreScanner.reseek(StoreScanner.java:1095) at org.apache.hadoop.hbase.regionserver.StoreScanner.seekAsDirection(StoreScanner.java:1084) at org.apache.hadoop.hbase.regionserver.StoreScanner.seekOrSkipToNextColumn(StoreScanner.java:821) at org.apache.hadoop.hbase.regionserver.StoreScanner.next(StoreScanner.java:684) at org.apache.hadoop.hbase.regionserver.KeyValueHeap.next(KeyValueHeap.java:155) at org.apache.hadoop.hbase.regionserver.HRegion$RegionScannerImpl.populateResult(HRegion.java:7399) at org.apache.hadoop.hbase.regionserver.HRegion$RegionScannerImpl.nextInternal(HRegion.java:7567) at org.apache.hadoop.hbase.regionserver.HRegion$RegionScannerImpl.nextRaw(HRegion.java:7331) at org.apache.hadoop.hbase.regionserver.HRegion$RegionScannerImpl.nextRaw(HRegion.java:7317) at org.apache.phoenix.coprocessor.PagedRegionScanner.next(PagedRegionScanner.java:57) at org.apache.phoenix.coprocessor.PagedRegionScanner.nextRaw(PagedRegionScanner.java:96) at org.apache.phoenix.iterate.RegionScannerFactory$1.nextRaw(RegionScannerFactory.java:211) at org.apache.phoenix.coprocessor.DelegateRegionScanner.nextRaw(DelegateRegionScanner.java:77) at org.apache.phoenix.coprocessor.DelegateRegionScanner.nextRaw(DelegateRegionScanner.java:77) at org.apache.phoenix.coprocessor.BaseScannerRegionObserver$RegionScannerHolder.nextRaw(BaseScannerRegionObserver.java:308) at org.apache.hadoop.hbase.regionserver.RSRpcServices.scan(RSRpcServices.java:3373) - locked <0x00007fc85f048ab0> (a org.apache.phoenix.coprocessor.BaseScannerRegionObserver$RegionScannerHolder) at org.apache.hadoop.hbase.regionserver.RSRpcServices.scan(RSRpcServices.java:3637) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45819) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:392) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:140) at org.apache.hadoop.hbase.ipc.RpcExecutor$Handler.run(RpcExecutor.java:359) at org.apache.hadoop.hbase.ipc.RpcExecutor$Handler.run(RpcExecutor.java:339) "RpcServer.default.FPBQ.Fifo.handler=20,queue=0,port=16020" #107 daemon prio=5 os_prio=0 tid=0x00007fc11d816800 nid=0x73e0 waiting on condition [0x00007fc1166f9000] java.lang.Thread.State: WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007fc1c9fec1d0> (a java.util.concurrent.Semaphore$NonfairSync) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer.parkAndCheckInterrupt(AbstractQueuedSynchronizer.java:836) at java.util.concurrent.locks.AbstractQueuedSynchronizer.doAcquireSharedInterruptibly(AbstractQueuedSynchronizer.java:997) at java.util.concurrent.locks.AbstractQueuedSynchronizer.acquireSharedInterruptibly(AbstractQueuedSynchronizer.java:1304) at java.util.concurrent.Semaphore.acquire(Semaphore.java:312) at org.apache.hadoop.hbase.ipc.FastPathBalancedQueueRpcExecutor$FastPathHandler.getCallRunner(FastPathBalancedQueueRpcExecutor.java:109) at org.apache.hadoop.hbase.ipc.RpcExecutor$Handler.run(RpcExecutor.java:339) "RpcServer.default.FPBQ.Fifo.handler=19,queue=9,port=16020" #106 daemon prio=5 os_prio=0 tid=0x00007fc11d814800 nid=0x73df waiting for monitor entry [0x00007fc1167f9000] java.lang.Thread.State: BLOCKED (on object monitor) at org.apache.hadoop.fs.FSInputStream.read(FSInputStream.java:73) - waiting to lock <0x00007fc4c2769660> (a org.apache.hadoop.fs.s3a.S3AInputStream) at org.apache.hadoop.fs.FSDataInputStream.read(FSDataInputStream.java:97) at org.apache.hadoop.hbase.io.FileLink$FileLinkInputStream.read(FileLink.java:171) at org.apache.hadoop.fs.FSDataInputStream.read(FSDataInputStream.java:97) at org.apache.hadoop.hbase.io.util.BlockIOUtils.preadWithExtra(BlockIOUtils.java:235) at org.apache.hadoop.hbase.io.hfile.HFileBlock$FSReaderImpl.readAtOffset(HFileBlock.java:1564) at org.apache.hadoop.hbase.io.hfile.HFileBlock$FSReaderImpl.readBlockDataInternal(HFileBlock.java:1788) at org.apache.hadoop.hbase.io.hfile.HFileBlock$FSReaderImpl.readBlockData(HFileBlock.java:1597) at org.apache.hadoop.hbase.io.hfile.HFileReaderImpl.readBlock(HFileReaderImpl.java:1347) at org.apache.hadoop.hbase.io.hfile.HFileReaderImpl.readBlock(HFileReaderImpl.java:1268) at org.apache.hadoop.hbase.io.hfile.HFileReaderImpl$HFileScannerImpl.readNextDataBlock(HFileReaderImpl.java:762) at org.apache.hadoop.hbase.io.hfile.HFileReaderImpl$EncodedScanner.next(HFileReaderImpl.java:1540) at org.apache.hadoop.hbase.regionserver.StoreFileScanner.reseekAtOrAfter(StoreFileScanner.java:351) at org.apache.hadoop.hbase.regionserver.StoreFileScanner.reseek(StoreFileScanner.java:244) at org.apache.hadoop.hbase.regionserver.NonLazyKeyValueScanner.doRealSeek(NonLazyKeyValueScanner.java:55) at org.apache.hadoop.hbase.regionserver.KeyValueHeap.generalizedSeek(KeyValueHeap.java:322) at org.apache.hadoop.hbase.regionserver.KeyValueHeap.reseek(KeyValueHeap.java:265) at org.apache.hadoop.hbase.regionserver.StoreScanner.reseek(StoreScanner.java:1095) at org.apache.hadoop.hbase.regionserver.StoreScanner.seekAsDirection(StoreScanner.java:1084) at org.apache.hadoop.hbase.regionserver.StoreScanner.seekOrSkipToNextColumn(StoreScanner.java:821) at org.apache.hadoop.hbase.regionserver.StoreScanner.next(StoreScanner.java:684) at org.apache.hadoop.hbase.regionserver.KeyValueHeap.next(KeyValueHeap.java:155) at org.apache.hadoop.hbase.regionserver.HRegion$RegionScannerImpl.populateResult(HRegion.java:7399) at org.apache.hadoop.hbase.regionserver.HRegion$RegionScannerImpl.nextInternal(HRegion.java:7567) at org.apache.hadoop.hbase.regionserver.HRegion$RegionScannerImpl.nextRaw(HRegion.java:7331) at org.apache.hadoop.hbase.regionserver.HRegion$RegionScannerImpl.nextRaw(HRegion.java:7317) at org.apache.phoenix.coprocessor.PagedRegionScanner.next(PagedRegionScanner.java:57) at org.apache.phoenix.coprocessor.PagedRegionScanner.nextRaw(PagedRegionScanner.java:96) at org.apache.phoenix.iterate.RegionScannerFactory$1.nextRaw(RegionScannerFactory.java:211) at org.apache.phoenix.coprocessor.DelegateRegionScanner.nextRaw(DelegateRegionScanner.java:77) at org.apache.phoenix.coprocessor.DelegateRegionScanner.nextRaw(DelegateRegionScanner.java:77) at org.apache.phoenix.coprocessor.BaseScannerRegionObserver$RegionScannerHolder.nextRaw(BaseScannerRegionObserver.java:308) at org.apache.hadoop.hbase.regionserver.RSRpcServices.scan(RSRpcServices.java:3373) - locked <0x00007fc85f09a740> (a org.apache.phoenix.coprocessor.BaseScannerRegionObserver$RegionScannerHolder) at org.apache.hadoop.hbase.regionserver.RSRpcServices.scan(RSRpcServices.java:3637) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45819) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:392) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:140) at org.apache.hadoop.hbase.ipc.RpcExecutor$Handler.run(RpcExecutor.java:359) at org.apache.hadoop.hbase.ipc.RpcExecutor$Handler.run(RpcExecutor.java:339) "RpcServer.default.FPBQ.Fifo.handler=18,queue=8,port=16020" #105 daemon prio=5 os_prio=0 tid=0x00007fc11d812800 nid=0x73de waiting for monitor entry [0x00007fc1168fa000] java.lang.Thread.State: BLOCKED (on object monitor) at org.apache.hadoop.fs.FSInputStream.read(FSInputStream.java:73) - waiting to lock <0x00007fc4c2769660> (a org.apache.hadoop.fs.s3a.S3AInputStream) at org.apache.hadoop.fs.FSDataInputStream.read(FSDataInputStream.java:97) at org.apache.hadoop.hbase.io.FileLink$FileLinkInputStream.read(FileLink.java:171) at org.apache.hadoop.fs.FSDataInputStream.read(FSDataInputStream.java:97) at org.apache.hadoop.hbase.io.util.BlockIOUtils.preadWithExtra(BlockIOUtils.java:235) at org.apache.hadoop.hbase.io.hfile.HFileBlock$FSReaderImpl.readAtOffset(HFileBlock.java:1564) at org.apache.hadoop.hbase.io.hfile.HFileBlock$FSReaderImpl.readBlockDataInternal(HFileBlock.java:1788) at org.apache.hadoop.hbase.io.hfile.HFileBlock$FSReaderImpl.readBlockData(HFileBlock.java:1597) at org.apache.hadoop.hbase.io.hfile.HFileReaderImpl.readBlock(HFileReaderImpl.java:1347) at org.apache.hadoop.hbase.io.hfile.HFileReaderImpl.readBlock(HFileReaderImpl.java:1268) at org.apache.hadoop.hbase.io.hfile.HFileBlockIndex$CellBasedKeyBlockIndexReader.loadDataBlockWithScanInfo(HFileBlockIndex.java:331) at org.apache.hadoop.hbase.io.hfile.HFileReaderImpl$HFileScannerImpl.seekTo(HFileReaderImpl.java:680) at org.apache.hadoop.hbase.io.hfile.HFileReaderImpl$HFileScannerImpl.reseekTo(HFileReaderImpl.java:662) at org.apache.hadoop.hbase.regionserver.StoreFileScanner.reseekAtOrAfter(StoreFileScanner.java:335) at org.apache.hadoop.hbase.regionserver.StoreFileScanner.reseek(StoreFileScanner.java:244) at org.apache.hadoop.hbase.regionserver.NonLazyKeyValueScanner.doRealSeek(NonLazyKeyValueScanner.java:55) at org.apache.hadoop.hbase.regionserver.KeyValueHeap.generalizedSeek(KeyValueHeap.java:322) at org.apache.hadoop.hbase.regionserver.KeyValueHeap.reseek(KeyValueHeap.java:265) at org.apache.hadoop.hbase.regionserver.StoreScanner.reseek(StoreScanner.java:1095) at org.apache.hadoop.hbase.regionserver.StoreScanner.seekToNextRow(StoreScanner.java:1073) at org.apache.hadoop.hbase.regionserver.StoreScanner.seekOrSkipToNextRow(StoreScanner.java:816) at org.apache.hadoop.hbase.regionserver.StoreScanner.next(StoreScanner.java:720) at org.apache.hadoop.hbase.regionserver.KeyValueHeap.next(KeyValueHeap.java:155) at org.apache.hadoop.hbase.regionserver.HRegion$RegionScannerImpl.populateResult(HRegion.java:7399) at org.apache.hadoop.hbase.regionserver.HRegion$RegionScannerImpl.nextInternal(HRegion.java:7567) at org.apache.hadoop.hbase.regionserver.HRegion$RegionScannerImpl.nextRaw(HRegion.java:7331) at org.apache.hadoop.hbase.regionserver.HRegion$RegionScannerImpl.nextRaw(HRegion.java:7317) at org.apache.phoenix.coprocessor.PagedRegionScanner.next(PagedRegionScanner.java:57) at org.apache.phoenix.coprocessor.PagedRegionScanner.nextRaw(PagedRegionScanner.java:96) at org.apache.phoenix.iterate.RegionScannerFactory$1.nextRaw(RegionScannerFactory.java:211) at org.apache.phoenix.coprocessor.DelegateRegionScanner.nextRaw(DelegateRegionScanner.java:77) at org.apache.phoenix.coprocessor.DelegateRegionScanner.nextRaw(DelegateRegionScanner.java:77) at org.apache.phoenix.coprocessor.BaseScannerRegionObserver$RegionScannerHolder.nextRaw(BaseScannerRegionObserver.java:308) at org.apache.hadoop.hbase.regionserver.RSRpcServices.scan(RSRpcServices.java:3373) - locked <0x00007fc85f08f3c8> (a org.apache.phoenix.coprocessor.BaseScannerRegionObserver$RegionScannerHolder) at org.apache.hadoop.hbase.regionserver.RSRpcServices.scan(RSRpcServices.java:3637) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45819) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:392) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:140) at org.apache.hadoop.hbase.ipc.RpcExecutor$Handler.run(RpcExecutor.java:359) at org.apache.hadoop.hbase.ipc.RpcExecutor$Handler.run(RpcExecutor.java:339) "RpcServer.default.FPBQ.Fifo.handler=17,queue=7,port=16020" #104 daemon prio=5 os_prio=0 tid=0x00007fc11d810800 nid=0x73dd waiting for monitor entry [0x00007fc1169fb000] java.lang.Thread.State: BLOCKED (on object monitor) at org.apache.hadoop.fs.FSInputStream.read(FSInputStream.java:73) - waiting to lock <0x00007fc4c2769660> (a org.apache.hadoop.fs.s3a.S3AInputStream) at org.apache.hadoop.fs.FSDataInputStream.read(FSDataInputStream.java:97) at org.apache.hadoop.hbase.io.FileLink$FileLinkInputStream.read(FileLink.java:171) at org.apache.hadoop.fs.FSDataInputStream.read(FSDataInputStream.java:97) at org.apache.hadoop.hbase.io.util.BlockIOUtils.preadWithExtra(BlockIOUtils.java:235) at org.apache.hadoop.hbase.io.hfile.HFileBlock$FSReaderImpl.readAtOffset(HFileBlock.java:1564) at org.apache.hadoop.hbase.io.hfile.HFileBlock$FSReaderImpl.readBlockDataInternal(HFileBlock.java:1788) at org.apache.hadoop.hbase.io.hfile.HFileBlock$FSReaderImpl.readBlockData(HFileBlock.java:1597) at org.apache.hadoop.hbase.io.hfile.HFileReaderImpl.readBlock(HFileReaderImpl.java:1347) at org.apache.hadoop.hbase.io.hfile.HFileReaderImpl.readBlock(HFileReaderImpl.java:1268) at org.apache.hadoop.hbase.io.hfile.HFileReaderImpl$HFileScannerImpl.readNextDataBlock(HFileReaderImpl.java:762) at org.apache.hadoop.hbase.io.hfile.HFileReaderImpl$EncodedScanner.next(HFileReaderImpl.java:1540) at org.apache.hadoop.hbase.regionserver.StoreFileScanner.reseekAtOrAfter(StoreFileScanner.java:351) at org.apache.hadoop.hbase.regionserver.StoreFileScanner.reseek(StoreFileScanner.java:244) at org.apache.hadoop.hbase.regionserver.NonLazyKeyValueScanner.doRealSeek(NonLazyKeyValueScanner.java:55) at org.apache.hadoop.hbase.regionserver.KeyValueHeap.generalizedSeek(KeyValueHeap.java:322) at org.apache.hadoop.hbase.regionserver.KeyValueHeap.reseek(KeyValueHeap.java:265) at org.apache.hadoop.hbase.regionserver.StoreScanner.reseek(StoreScanner.java:1095) at org.apache.hadoop.hbase.regionserver.StoreScanner.seekAsDirection(StoreScanner.java:1084) at org.apache.hadoop.hbase.regionserver.StoreScanner.seekOrSkipToNextColumn(StoreScanner.java:821) at org.apache.hadoop.hbase.regionserver.StoreScanner.next(StoreScanner.java:684) at org.apache.hadoop.hbase.regionserver.KeyValueHeap.next(KeyValueHeap.java:155) at org.apache.hadoop.hbase.regionserver.HRegion$RegionScannerImpl.populateResult(HRegion.java:7399) at org.apache.hadoop.hbase.regionserver.HRegion$RegionScannerImpl.nextInternal(HRegion.java:7567) at org.apache.hadoop.hbase.regionserver.HRegion$RegionScannerImpl.nextRaw(HRegion.java:7331) at org.apache.hadoop.hbase.regionserver.HRegion$RegionScannerImpl.nextRaw(HRegion.java:7317) at org.apache.phoenix.coprocessor.PagedRegionScanner.next(PagedRegionScanner.java:57) at org.apache.phoenix.coprocessor.PagedRegionScanner.nextRaw(PagedRegionScanner.java:96) at org.apache.phoenix.iterate.RegionScannerFactory$1.nextRaw(RegionScannerFactory.java:211) at org.apache.phoenix.coprocessor.DelegateRegionScanner.nextRaw(DelegateRegionScanner.java:77) at org.apache.phoenix.coprocessor.DelegateRegionScanner.nextRaw(DelegateRegionScanner.java:77) at org.apache.phoenix.coprocessor.BaseScannerRegionObserver$RegionScannerHolder.nextRaw(BaseScannerRegionObserver.java:308) at org.apache.hadoop.hbase.regionserver.RSRpcServices.scan(RSRpcServices.java:3373) - locked <0x00007fc85f0b7320> (a org.apache.phoenix.coprocessor.BaseScannerRegionObserver$RegionScannerHolder) at org.apache.hadoop.hbase.regionserver.RSRpcServices.scan(RSRpcServices.java:3637) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45819) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:392) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:140) at org.apache.hadoop.hbase.ipc.RpcExecutor$Handler.run(RpcExecutor.java:359) at org.apache.hadoop.hbase.ipc.RpcExecutor$Handler.run(RpcExecutor.java:339) "RpcServer.default.FPBQ.Fifo.handler=16,queue=6,port=16020" #103 daemon prio=5 os_prio=0 tid=0x00007fc11d80e000 nid=0x73dc waiting on condition [0x00007fc116afd000] java.lang.Thread.State: WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007fc1c9ff5070> (a java.util.concurrent.Semaphore$NonfairSync) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer.parkAndCheckInterrupt(AbstractQueuedSynchronizer.java:836) at java.util.concurrent.locks.AbstractQueuedSynchronizer.doAcquireSharedInterruptibly(AbstractQueuedSynchronizer.java:997) at java.util.concurrent.locks.AbstractQueuedSynchronizer.acquireSharedInterruptibly(AbstractQueuedSynchronizer.java:1304) at java.util.concurrent.Semaphore.acquire(Semaphore.java:312) at org.apache.hadoop.hbase.ipc.FastPathBalancedQueueRpcExecutor$FastPathHandler.getCallRunner(FastPathBalancedQueueRpcExecutor.java:109) at org.apache.hadoop.hbase.ipc.RpcExecutor$Handler.run(RpcExecutor.java:339) "RpcServer.default.FPBQ.Fifo.handler=15,queue=5,port=16020" #102 daemon prio=5 os_prio=0 tid=0x00007fc11d80c000 nid=0x73db waiting for monitor entry [0x00007fc116bfd000] java.lang.Thread.State: BLOCKED (on object monitor) at org.apache.hadoop.fs.FSInputStream.read(FSInputStream.java:73) - waiting to lock <0x00007fc4c2769660> (a org.apache.hadoop.fs.s3a.S3AInputStream) at org.apache.hadoop.fs.FSDataInputStream.read(FSDataInputStream.java:97) at org.apache.hadoop.hbase.io.FileLink$FileLinkInputStream.read(FileLink.java:171) at org.apache.hadoop.fs.FSDataInputStream.read(FSDataInputStream.java:97) at org.apache.hadoop.hbase.io.util.BlockIOUtils.preadWithExtra(BlockIOUtils.java:235) at org.apache.hadoop.hbase.io.hfile.HFileBlock$FSReaderImpl.readAtOffset(HFileBlock.java:1564) at org.apache.hadoop.hbase.io.hfile.HFileBlock$FSReaderImpl.readBlockDataInternal(HFileBlock.java:1788) at org.apache.hadoop.hbase.io.hfile.HFileBlock$FSReaderImpl.readBlockData(HFileBlock.java:1597) at org.apache.hadoop.hbase.io.hfile.HFileReaderImpl.readBlock(HFileReaderImpl.java:1347) at org.apache.hadoop.hbase.io.hfile.HFileReaderImpl.readBlock(HFileReaderImpl.java:1268) at org.apache.hadoop.hbase.io.hfile.HFileBlockIndex$CellBasedKeyBlockIndexReader.loadDataBlockWithScanInfo(HFileBlockIndex.java:331) at org.apache.hadoop.hbase.io.hfile.HFileReaderImpl$HFileScannerImpl.seekTo(HFileReaderImpl.java:680) at org.apache.hadoop.hbase.io.hfile.HFileReaderImpl$HFileScannerImpl.seekTo(HFileReaderImpl.java:632) at org.apache.hadoop.hbase.regionserver.StoreFileScanner.seekAtOrAfter(StoreFileScanner.java:315) at org.apache.hadoop.hbase.regionserver.StoreFileScanner.seek(StoreFileScanner.java:216) at org.apache.hadoop.hbase.regionserver.StoreScanner.seekScanners(StoreScanner.java:417) at org.apache.hadoop.hbase.regionserver.StoreScanner.(StoreScanner.java:257) at org.apache.hadoop.hbase.regionserver.HStore.createScanner(HStore.java:1824) at org.apache.hadoop.hbase.regionserver.HStore.getScanner(HStore.java:1815) at org.apache.hadoop.hbase.regionserver.HRegion$RegionScannerImpl.initializeScanners(HRegion.java:7221) at org.apache.hadoop.hbase.regionserver.HRegion$RegionScannerImpl.(HRegion.java:7201) at org.apache.hadoop.hbase.regionserver.HRegion.instantiateRegionScanner(HRegion.java:3159) at org.apache.hadoop.hbase.regionserver.HRegion.getScanner(HRegion.java:3139) at org.apache.hadoop.hbase.regionserver.HRegion.getScanner(HRegion.java:3121) at org.apache.hadoop.hbase.regionserver.HRegion.getScanner(HRegion.java:3115) at org.apache.hadoop.hbase.regionserver.RSRpcServices.newRegionScanner(RSRpcServices.java:3201) at org.apache.hadoop.hbase.regionserver.RSRpcServices.scan(RSRpcServices.java:3549) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45819) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:392) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:140) at org.apache.hadoop.hbase.ipc.RpcExecutor$Handler.run(RpcExecutor.java:359) at org.apache.hadoop.hbase.ipc.RpcExecutor$Handler.run(RpcExecutor.java:339) "RpcServer.default.FPBQ.Fifo.handler=14,queue=4,port=16020" #101 daemon prio=5 os_prio=0 tid=0x00007fc11d80a000 nid=0x73da waiting for monitor entry [0x00007fc116cfe000] java.lang.Thread.State: BLOCKED (on object monitor) at org.apache.hadoop.fs.FSInputStream.read(FSInputStream.java:73) - waiting to lock <0x00007fc4c2769660> (a org.apache.hadoop.fs.s3a.S3AInputStream) at org.apache.hadoop.fs.FSDataInputStream.read(FSDataInputStream.java:97) at org.apache.hadoop.hbase.io.FileLink$FileLinkInputStream.read(FileLink.java:171) at org.apache.hadoop.fs.FSDataInputStream.read(FSDataInputStream.java:97) at org.apache.hadoop.hbase.io.util.BlockIOUtils.preadWithExtra(BlockIOUtils.java:235) at org.apache.hadoop.hbase.io.hfile.HFileBlock$FSReaderImpl.readAtOffset(HFileBlock.java:1564) at org.apache.hadoop.hbase.io.hfile.HFileBlock$FSReaderImpl.readBlockDataInternal(HFileBlock.java:1788) at org.apache.hadoop.hbase.io.hfile.HFileBlock$FSReaderImpl.readBlockData(HFileBlock.java:1597) at org.apache.hadoop.hbase.io.hfile.HFileReaderImpl.readBlock(HFileReaderImpl.java:1347) at org.apache.hadoop.hbase.io.hfile.HFileReaderImpl.readBlock(HFileReaderImpl.java:1268) at org.apache.hadoop.hbase.io.hfile.HFileReaderImpl$HFileScannerImpl.readNextDataBlock(HFileReaderImpl.java:762) at org.apache.hadoop.hbase.io.hfile.HFileReaderImpl$EncodedScanner.next(HFileReaderImpl.java:1540) at org.apache.hadoop.hbase.regionserver.StoreFileScanner.reseekAtOrAfter(StoreFileScanner.java:351) at org.apache.hadoop.hbase.regionserver.StoreFileScanner.reseek(StoreFileScanner.java:244) at org.apache.hadoop.hbase.regionserver.NonLazyKeyValueScanner.doRealSeek(NonLazyKeyValueScanner.java:55) at org.apache.hadoop.hbase.regionserver.KeyValueHeap.generalizedSeek(KeyValueHeap.java:322) at org.apache.hadoop.hbase.regionserver.KeyValueHeap.reseek(KeyValueHeap.java:265) at org.apache.hadoop.hbase.regionserver.StoreScanner.reseek(StoreScanner.java:1095) at org.apache.hadoop.hbase.regionserver.StoreScanner.seekAsDirection(StoreScanner.java:1084) at org.apache.hadoop.hbase.regionserver.StoreScanner.seekOrSkipToNextColumn(StoreScanner.java:821) at org.apache.hadoop.hbase.regionserver.StoreScanner.next(StoreScanner.java:684) at org.apache.hadoop.hbase.regionserver.KeyValueHeap.next(KeyValueHeap.java:155) at org.apache.hadoop.hbase.regionserver.HRegion$RegionScannerImpl.populateResult(HRegion.java:7399) at org.apache.hadoop.hbase.regionserver.HRegion$RegionScannerImpl.nextInternal(HRegion.java:7567) at org.apache.hadoop.hbase.regionserver.HRegion$RegionScannerImpl.nextRaw(HRegion.java:7331) at org.apache.hadoop.hbase.regionserver.HRegion$RegionScannerImpl.nextRaw(HRegion.java:7317) at org.apache.phoenix.coprocessor.PagedRegionScanner.next(PagedRegionScanner.java:57) at org.apache.phoenix.coprocessor.PagedRegionScanner.nextRaw(PagedRegionScanner.java:96) at org.apache.phoenix.iterate.RegionScannerFactory$1.nextRaw(RegionScannerFactory.java:211) at org.apache.phoenix.coprocessor.DelegateRegionScanner.nextRaw(DelegateRegionScanner.java:77) at org.apache.phoenix.coprocessor.DelegateRegionScanner.nextRaw(DelegateRegionScanner.java:77) at org.apache.phoenix.coprocessor.BaseScannerRegionObserver$RegionScannerHolder.nextRaw(BaseScannerRegionObserver.java:308) at org.apache.hadoop.hbase.regionserver.RSRpcServices.scan(RSRpcServices.java:3373) - locked <0x00007fc6b13a11b8> (a org.apache.phoenix.coprocessor.BaseScannerRegionObserver$RegionScannerHolder) at org.apache.hadoop.hbase.regionserver.RSRpcServices.scan(RSRpcServices.java:3637) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45819) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:392) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:140) at org.apache.hadoop.hbase.ipc.RpcExecutor$Handler.run(RpcExecutor.java:359) at org.apache.hadoop.hbase.ipc.RpcExecutor$Handler.run(RpcExecutor.java:339) "RpcServer.default.FPBQ.Fifo.handler=13,queue=3,port=16020" #100 daemon prio=5 os_prio=0 tid=0x00007fc11d808000 nid=0x73d9 waiting for monitor entry [0x00007fc116dff000] java.lang.Thread.State: BLOCKED (on object monitor) at org.apache.hadoop.fs.FSInputStream.read(FSInputStream.java:73) - waiting to lock <0x00007fc4c2769660> (a org.apache.hadoop.fs.s3a.S3AInputStream) at org.apache.hadoop.fs.FSDataInputStream.read(FSDataInputStream.java:97) at org.apache.hadoop.hbase.io.FileLink$FileLinkInputStream.read(FileLink.java:171) at org.apache.hadoop.fs.FSDataInputStream.read(FSDataInputStream.java:97) at org.apache.hadoop.hbase.io.util.BlockIOUtils.preadWithExtra(BlockIOUtils.java:235) at org.apache.hadoop.hbase.io.hfile.HFileBlock$FSReaderImpl.readAtOffset(HFileBlock.java:1564) at org.apache.hadoop.hbase.io.hfile.HFileBlock$FSReaderImpl.readBlockDataInternal(HFileBlock.java:1788) at org.apache.hadoop.hbase.io.hfile.HFileBlock$FSReaderImpl.readBlockData(HFileBlock.java:1597) at org.apache.hadoop.hbase.io.hfile.HFileReaderImpl.readBlock(HFileReaderImpl.java:1347) at org.apache.hadoop.hbase.io.hfile.HFileReaderImpl.readBlock(HFileReaderImpl.java:1268) at org.apache.hadoop.hbase.io.hfile.HFileReaderImpl$HFileScannerImpl.readNextDataBlock(HFileReaderImpl.java:762) at org.apache.hadoop.hbase.io.hfile.HFileReaderImpl$EncodedScanner.next(HFileReaderImpl.java:1540) at org.apache.hadoop.hbase.regionserver.StoreFileScanner.reseekAtOrAfter(StoreFileScanner.java:351) at org.apache.hadoop.hbase.regionserver.StoreFileScanner.reseek(StoreFileScanner.java:244) at org.apache.hadoop.hbase.regionserver.NonLazyKeyValueScanner.doRealSeek(NonLazyKeyValueScanner.java:55) at org.apache.hadoop.hbase.regionserver.KeyValueHeap.generalizedSeek(KeyValueHeap.java:322) at org.apache.hadoop.hbase.regionserver.KeyValueHeap.reseek(KeyValueHeap.java:265) at org.apache.hadoop.hbase.regionserver.StoreScanner.reseek(StoreScanner.java:1095) at org.apache.hadoop.hbase.regionserver.StoreScanner.seekAsDirection(StoreScanner.java:1084) at org.apache.hadoop.hbase.regionserver.StoreScanner.seekOrSkipToNextColumn(StoreScanner.java:821) at org.apache.hadoop.hbase.regionserver.StoreScanner.next(StoreScanner.java:684) at org.apache.hadoop.hbase.regionserver.KeyValueHeap.next(KeyValueHeap.java:155) at org.apache.hadoop.hbase.regionserver.HRegion$RegionScannerImpl.populateResult(HRegion.java:7399) at org.apache.hadoop.hbase.regionserver.HRegion$RegionScannerImpl.nextInternal(HRegion.java:7567) at org.apache.hadoop.hbase.regionserver.HRegion$RegionScannerImpl.nextRaw(HRegion.java:7331) at org.apache.hadoop.hbase.regionserver.HRegion$RegionScannerImpl.nextRaw(HRegion.java:7317) at org.apache.phoenix.coprocessor.PagedRegionScanner.next(PagedRegionScanner.java:57) at org.apache.phoenix.coprocessor.PagedRegionScanner.nextRaw(PagedRegionScanner.java:96) at org.apache.phoenix.iterate.RegionScannerFactory$1.nextRaw(RegionScannerFactory.java:211) at org.apache.phoenix.coprocessor.DelegateRegionScanner.nextRaw(DelegateRegionScanner.java:77) at org.apache.phoenix.coprocessor.DelegateRegionScanner.nextRaw(DelegateRegionScanner.java:77) at org.apache.phoenix.coprocessor.BaseScannerRegionObserver$RegionScannerHolder.nextRaw(BaseScannerRegionObserver.java:308) at org.apache.hadoop.hbase.regionserver.RSRpcServices.scan(RSRpcServices.java:3373) - locked <0x00007fc85f08f9d0> (a org.apache.phoenix.coprocessor.BaseScannerRegionObserver$RegionScannerHolder) at org.apache.hadoop.hbase.regionserver.RSRpcServices.scan(RSRpcServices.java:3637) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45819) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:392) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:140) at org.apache.hadoop.hbase.ipc.RpcExecutor$Handler.run(RpcExecutor.java:359) at org.apache.hadoop.hbase.ipc.RpcExecutor$Handler.run(RpcExecutor.java:339) "RpcServer.default.FPBQ.Fifo.handler=12,queue=2,port=16020" #99 daemon prio=5 os_prio=0 tid=0x00007fc11d806800 nid=0x73d8 waiting for monitor entry [0x00007fc116f00000] java.lang.Thread.State: BLOCKED (on object monitor) at org.apache.hadoop.fs.FSInputStream.read(FSInputStream.java:73) - waiting to lock <0x00007fc4c2769660> (a org.apache.hadoop.fs.s3a.S3AInputStream) at org.apache.hadoop.fs.FSDataInputStream.read(FSDataInputStream.java:97) at org.apache.hadoop.hbase.io.FileLink$FileLinkInputStream.read(FileLink.java:171) at org.apache.hadoop.fs.FSDataInputStream.read(FSDataInputStream.java:97) at org.apache.hadoop.hbase.io.util.BlockIOUtils.preadWithExtra(BlockIOUtils.java:235) at org.apache.hadoop.hbase.io.hfile.HFileBlock$FSReaderImpl.readAtOffset(HFileBlock.java:1564) at org.apache.hadoop.hbase.io.hfile.HFileBlock$FSReaderImpl.readBlockDataInternal(HFileBlock.java:1788) at org.apache.hadoop.hbase.io.hfile.HFileBlock$FSReaderImpl.readBlockData(HFileBlock.java:1597) at org.apache.hadoop.hbase.io.hfile.HFileReaderImpl.readBlock(HFileReaderImpl.java:1347) at org.apache.hadoop.hbase.io.hfile.HFileReaderImpl.readBlock(HFileReaderImpl.java:1268) at org.apache.hadoop.hbase.io.hfile.HFileReaderImpl$HFileScannerImpl.readNextDataBlock(HFileReaderImpl.java:762) at org.apache.hadoop.hbase.io.hfile.HFileReaderImpl$EncodedScanner.next(HFileReaderImpl.java:1540) at org.apache.hadoop.hbase.regionserver.StoreFileScanner.next(StoreFileScanner.java:196) at org.apache.hadoop.hbase.regionserver.KeyValueHeap.next(KeyValueHeap.java:122) at org.apache.hadoop.hbase.regionserver.StoreScanner.trySkipToNextRow(StoreScanner.java:884) at org.apache.hadoop.hbase.regionserver.StoreScanner.seekOrSkipToNextRow(StoreScanner.java:812) at org.apache.hadoop.hbase.regionserver.StoreScanner.next(StoreScanner.java:720) at org.apache.hadoop.hbase.regionserver.KeyValueHeap.next(KeyValueHeap.java:155) at org.apache.hadoop.hbase.regionserver.HRegion$RegionScannerImpl.populateResult(HRegion.java:7399) at org.apache.hadoop.hbase.regionserver.HRegion$RegionScannerImpl.nextInternal(HRegion.java:7567) at org.apache.hadoop.hbase.regionserver.HRegion$RegionScannerImpl.nextRaw(HRegion.java:7331) at org.apache.hadoop.hbase.regionserver.HRegion$RegionScannerImpl.nextRaw(HRegion.java:7317) at org.apache.phoenix.coprocessor.PagedRegionScanner.next(PagedRegionScanner.java:57) at org.apache.phoenix.coprocessor.PagedRegionScanner.nextRaw(PagedRegionScanner.java:96) at org.apache.phoenix.iterate.RegionScannerFactory$1.nextRaw(RegionScannerFactory.java:211) at org.apache.phoenix.coprocessor.DelegateRegionScanner.nextRaw(DelegateRegionScanner.java:77) at org.apache.phoenix.coprocessor.DelegateRegionScanner.nextRaw(DelegateRegionScanner.java:77) at org.apache.phoenix.coprocessor.BaseScannerRegionObserver$RegionScannerHolder.nextRaw(BaseScannerRegionObserver.java:308) at org.apache.hadoop.hbase.regionserver.RSRpcServices.scan(RSRpcServices.java:3373) - locked <0x00007fc85f0b9480> (a org.apache.phoenix.coprocessor.BaseScannerRegionObserver$RegionScannerHolder) at org.apache.hadoop.hbase.regionserver.RSRpcServices.scan(RSRpcServices.java:3637) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45819) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:392) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:140) at org.apache.hadoop.hbase.ipc.RpcExecutor$Handler.run(RpcExecutor.java:359) at org.apache.hadoop.hbase.ipc.RpcExecutor$Handler.run(RpcExecutor.java:339) "RpcServer.default.FPBQ.Fifo.handler=11,queue=1,port=16020" #98 daemon prio=5 os_prio=0 tid=0x00007fc11d804000 nid=0x73d7 waiting for monitor entry [0x00007fc117001000] java.lang.Thread.State: BLOCKED (on object monitor) at org.apache.hadoop.fs.FSInputStream.read(FSInputStream.java:73) - waiting to lock <0x00007fc4c2769660> (a org.apache.hadoop.fs.s3a.S3AInputStream) at org.apache.hadoop.fs.FSDataInputStream.read(FSDataInputStream.java:97) at org.apache.hadoop.hbase.io.FileLink$FileLinkInputStream.read(FileLink.java:171) at org.apache.hadoop.fs.FSDataInputStream.read(FSDataInputStream.java:97) at org.apache.hadoop.hbase.io.util.BlockIOUtils.preadWithExtra(BlockIOUtils.java:235) at org.apache.hadoop.hbase.io.hfile.HFileBlock$FSReaderImpl.readAtOffset(HFileBlock.java:1564) at org.apache.hadoop.hbase.io.hfile.HFileBlock$FSReaderImpl.readBlockDataInternal(HFileBlock.java:1788) at org.apache.hadoop.hbase.io.hfile.HFileBlock$FSReaderImpl.readBlockData(HFileBlock.java:1597) at org.apache.hadoop.hbase.io.hfile.HFileReaderImpl.readBlock(HFileReaderImpl.java:1347) at org.apache.hadoop.hbase.io.hfile.HFileReaderImpl.readBlock(HFileReaderImpl.java:1268) at org.apache.hadoop.hbase.io.hfile.HFileReaderImpl$HFileScannerImpl.readNextDataBlock(HFileReaderImpl.java:762) at org.apache.hadoop.hbase.io.hfile.HFileReaderImpl$EncodedScanner.next(HFileReaderImpl.java:1540) at org.apache.hadoop.hbase.regionserver.StoreFileScanner.reseekAtOrAfter(StoreFileScanner.java:351) at org.apache.hadoop.hbase.regionserver.StoreFileScanner.reseek(StoreFileScanner.java:244) at org.apache.hadoop.hbase.regionserver.NonLazyKeyValueScanner.doRealSeek(NonLazyKeyValueScanner.java:55) at org.apache.hadoop.hbase.regionserver.KeyValueHeap.generalizedSeek(KeyValueHeap.java:322) at org.apache.hadoop.hbase.regionserver.KeyValueHeap.reseek(KeyValueHeap.java:265) at org.apache.hadoop.hbase.regionserver.StoreScanner.reseek(StoreScanner.java:1095) at org.apache.hadoop.hbase.regionserver.StoreScanner.seekAsDirection(StoreScanner.java:1084) at org.apache.hadoop.hbase.regionserver.StoreScanner.seekOrSkipToNextColumn(StoreScanner.java:821) at org.apache.hadoop.hbase.regionserver.StoreScanner.next(StoreScanner.java:684) at org.apache.hadoop.hbase.regionserver.KeyValueHeap.next(KeyValueHeap.java:155) at org.apache.hadoop.hbase.regionserver.HRegion$RegionScannerImpl.populateResult(HRegion.java:7399) at org.apache.hadoop.hbase.regionserver.HRegion$RegionScannerImpl.nextInternal(HRegion.java:7567) at org.apache.hadoop.hbase.regionserver.HRegion$RegionScannerImpl.nextRaw(HRegion.java:7331) at org.apache.hadoop.hbase.regionserver.HRegion$RegionScannerImpl.nextRaw(HRegion.java:7317) at org.apache.phoenix.coprocessor.PagedRegionScanner.next(PagedRegionScanner.java:57) at org.apache.phoenix.coprocessor.PagedRegionScanner.nextRaw(PagedRegionScanner.java:96) at org.apache.phoenix.iterate.RegionScannerFactory$1.nextRaw(RegionScannerFactory.java:211) at org.apache.phoenix.coprocessor.DelegateRegionScanner.nextRaw(DelegateRegionScanner.java:77) at org.apache.phoenix.coprocessor.DelegateRegionScanner.nextRaw(DelegateRegionScanner.java:77) at org.apache.phoenix.coprocessor.BaseScannerRegionObserver$RegionScannerHolder.nextRaw(BaseScannerRegionObserver.java:308) at org.apache.hadoop.hbase.regionserver.RSRpcServices.scan(RSRpcServices.java:3373) - locked <0x00007fc85f0495c8> (a org.apache.phoenix.coprocessor.BaseScannerRegionObserver$RegionScannerHolder) at org.apache.hadoop.hbase.regionserver.RSRpcServices.scan(RSRpcServices.java:3637) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45819) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:392) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:140) at org.apache.hadoop.hbase.ipc.RpcExecutor$Handler.run(RpcExecutor.java:359) at org.apache.hadoop.hbase.ipc.RpcExecutor$Handler.run(RpcExecutor.java:339) "RpcServer.default.FPBQ.Fifo.handler=10,queue=0,port=16020" #97 daemon prio=5 os_prio=0 tid=0x00007fc11d802000 nid=0x73d6 waiting for monitor entry [0x00007fc117102000] java.lang.Thread.State: BLOCKED (on object monitor) at org.apache.hadoop.fs.FSInputStream.read(FSInputStream.java:73) - waiting to lock <0x00007fc4c2769660> (a org.apache.hadoop.fs.s3a.S3AInputStream) at org.apache.hadoop.fs.FSDataInputStream.read(FSDataInputStream.java:97) at org.apache.hadoop.hbase.io.FileLink$FileLinkInputStream.read(FileLink.java:171) at org.apache.hadoop.fs.FSDataInputStream.read(FSDataInputStream.java:97) at org.apache.hadoop.hbase.io.util.BlockIOUtils.preadWithExtra(BlockIOUtils.java:235) at org.apache.hadoop.hbase.io.hfile.HFileBlock$FSReaderImpl.readAtOffset(HFileBlock.java:1564) at org.apache.hadoop.hbase.io.hfile.HFileBlock$FSReaderImpl.readBlockDataInternal(HFileBlock.java:1788) at org.apache.hadoop.hbase.io.hfile.HFileBlock$FSReaderImpl.readBlockData(HFileBlock.java:1597) at org.apache.hadoop.hbase.io.hfile.HFileReaderImpl.readBlock(HFileReaderImpl.java:1347) at org.apache.hadoop.hbase.io.hfile.HFileReaderImpl.readBlock(HFileReaderImpl.java:1268) at org.apache.hadoop.hbase.io.hfile.HFileReaderImpl$HFileScannerImpl.readNextDataBlock(HFileReaderImpl.java:762) at org.apache.hadoop.hbase.io.hfile.HFileReaderImpl$EncodedScanner.next(HFileReaderImpl.java:1540) at org.apache.hadoop.hbase.regionserver.StoreFileScanner.reseekAtOrAfter(StoreFileScanner.java:351) at org.apache.hadoop.hbase.regionserver.StoreFileScanner.reseek(StoreFileScanner.java:244) at org.apache.hadoop.hbase.regionserver.NonLazyKeyValueScanner.doRealSeek(NonLazyKeyValueScanner.java:55) at org.apache.hadoop.hbase.regionserver.KeyValueHeap.generalizedSeek(KeyValueHeap.java:322) at org.apache.hadoop.hbase.regionserver.KeyValueHeap.reseek(KeyValueHeap.java:265) at org.apache.hadoop.hbase.regionserver.StoreScanner.reseek(StoreScanner.java:1095) at org.apache.hadoop.hbase.regionserver.StoreScanner.seekAsDirection(StoreScanner.java:1084) at org.apache.hadoop.hbase.regionserver.StoreScanner.seekOrSkipToNextColumn(StoreScanner.java:821) at org.apache.hadoop.hbase.regionserver.StoreScanner.next(StoreScanner.java:684) at org.apache.hadoop.hbase.regionserver.KeyValueHeap.next(KeyValueHeap.java:155) at org.apache.hadoop.hbase.regionserver.HRegion$RegionScannerImpl.populateResult(HRegion.java:7399) at org.apache.hadoop.hbase.regionserver.HRegion$RegionScannerImpl.nextInternal(HRegion.java:7567) at org.apache.hadoop.hbase.regionserver.HRegion$RegionScannerImpl.nextRaw(HRegion.java:7331) at org.apache.hadoop.hbase.regionserver.HRegion$RegionScannerImpl.nextRaw(HRegion.java:7317) at org.apache.phoenix.coprocessor.PagedRegionScanner.next(PagedRegionScanner.java:57) at org.apache.phoenix.coprocessor.PagedRegionScanner.nextRaw(PagedRegionScanner.java:96) at org.apache.phoenix.iterate.RegionScannerFactory$1.nextRaw(RegionScannerFactory.java:211) at org.apache.phoenix.coprocessor.DelegateRegionScanner.nextRaw(DelegateRegionScanner.java:77) at org.apache.phoenix.coprocessor.DelegateRegionScanner.nextRaw(DelegateRegionScanner.java:77) at org.apache.phoenix.coprocessor.BaseScannerRegionObserver$RegionScannerHolder.nextRaw(BaseScannerRegionObserver.java:308) at org.apache.hadoop.hbase.regionserver.RSRpcServices.scan(RSRpcServices.java:3373) - locked <0x00007fc85f09fa78> (a org.apache.phoenix.coprocessor.BaseScannerRegionObserver$RegionScannerHolder) at org.apache.hadoop.hbase.regionserver.RSRpcServices.scan(RSRpcServices.java:3637) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45819) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:392) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:140) at org.apache.hadoop.hbase.ipc.RpcExecutor$Handler.run(RpcExecutor.java:359) at org.apache.hadoop.hbase.ipc.RpcExecutor$Handler.run(RpcExecutor.java:339) "RpcServer.default.FPBQ.Fifo.handler=9,queue=9,port=16020" #96 daemon prio=5 os_prio=0 tid=0x00007fc11d800800 nid=0x73d5 waiting for monitor entry [0x00007fc117203000] java.lang.Thread.State: BLOCKED (on object monitor) at org.apache.hadoop.fs.FSInputStream.read(FSInputStream.java:73) - waiting to lock <0x00007fc4c2769660> (a org.apache.hadoop.fs.s3a.S3AInputStream) at org.apache.hadoop.fs.FSDataInputStream.read(FSDataInputStream.java:97) at org.apache.hadoop.hbase.io.FileLink$FileLinkInputStream.read(FileLink.java:171) at org.apache.hadoop.fs.FSDataInputStream.read(FSDataInputStream.java:97) at org.apache.hadoop.hbase.io.util.BlockIOUtils.preadWithExtra(BlockIOUtils.java:235) at org.apache.hadoop.hbase.io.hfile.HFileBlock$FSReaderImpl.readAtOffset(HFileBlock.java:1564) at org.apache.hadoop.hbase.io.hfile.HFileBlock$FSReaderImpl.readBlockDataInternal(HFileBlock.java:1788) at org.apache.hadoop.hbase.io.hfile.HFileBlock$FSReaderImpl.readBlockData(HFileBlock.java:1597) at org.apache.hadoop.hbase.io.hfile.HFileReaderImpl.readBlock(HFileReaderImpl.java:1347) at org.apache.hadoop.hbase.io.hfile.HFileReaderImpl.readBlock(HFileReaderImpl.java:1268) at org.apache.hadoop.hbase.io.hfile.HFileBlockIndex$CellBasedKeyBlockIndexReader.loadDataBlockWithScanInfo(HFileBlockIndex.java:331) at org.apache.hadoop.hbase.io.hfile.HFileReaderImpl$HFileScannerImpl.seekTo(HFileReaderImpl.java:680) at org.apache.hadoop.hbase.io.hfile.HFileReaderImpl$HFileScannerImpl.reseekTo(HFileReaderImpl.java:662) at org.apache.hadoop.hbase.regionserver.StoreFileScanner.reseekAtOrAfter(StoreFileScanner.java:335) at org.apache.hadoop.hbase.regionserver.StoreFileScanner.reseek(StoreFileScanner.java:244) at org.apache.hadoop.hbase.regionserver.NonLazyKeyValueScanner.doRealSeek(NonLazyKeyValueScanner.java:55) at org.apache.hadoop.hbase.regionserver.KeyValueHeap.generalizedSeek(KeyValueHeap.java:322) at org.apache.hadoop.hbase.regionserver.KeyValueHeap.reseek(KeyValueHeap.java:265) at org.apache.hadoop.hbase.regionserver.StoreScanner.reseek(StoreScanner.java:1095) at org.apache.hadoop.hbase.regionserver.StoreScanner.seekToNextRow(StoreScanner.java:1073) at org.apache.hadoop.hbase.regionserver.StoreScanner.seekOrSkipToNextRow(StoreScanner.java:816) at org.apache.hadoop.hbase.regionserver.StoreScanner.next(StoreScanner.java:720) at org.apache.hadoop.hbase.regionserver.KeyValueHeap.next(KeyValueHeap.java:155) at org.apache.hadoop.hbase.regionserver.HRegion$RegionScannerImpl.populateResult(HRegion.java:7399) at org.apache.hadoop.hbase.regionserver.HRegion$RegionScannerImpl.nextInternal(HRegion.java:7567) at org.apache.hadoop.hbase.regionserver.HRegion$RegionScannerImpl.nextRaw(HRegion.java:7331) at org.apache.hadoop.hbase.regionserver.HRegion$RegionScannerImpl.nextRaw(HRegion.java:7317) at org.apache.phoenix.coprocessor.PagedRegionScanner.next(PagedRegionScanner.java:57) at org.apache.phoenix.coprocessor.PagedRegionScanner.nextRaw(PagedRegionScanner.java:96) at org.apache.phoenix.iterate.RegionScannerFactory$1.nextRaw(RegionScannerFactory.java:211) at org.apache.phoenix.coprocessor.DelegateRegionScanner.nextRaw(DelegateRegionScanner.java:77) at org.apache.phoenix.coprocessor.DelegateRegionScanner.nextRaw(DelegateRegionScanner.java:77) at org.apache.phoenix.coprocessor.BaseScannerRegionObserver$RegionScannerHolder.nextRaw(BaseScannerRegionObserver.java:308) at org.apache.hadoop.hbase.regionserver.RSRpcServices.scan(RSRpcServices.java:3373) - locked <0x00007fc68ab4a9f8> (a org.apache.phoenix.coprocessor.BaseScannerRegionObserver$RegionScannerHolder) at org.apache.hadoop.hbase.regionserver.RSRpcServices.scan(RSRpcServices.java:3637) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45819) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:392) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:140) at org.apache.hadoop.hbase.ipc.RpcExecutor$Handler.run(RpcExecutor.java:359) at org.apache.hadoop.hbase.ipc.RpcExecutor$Handler.run(RpcExecutor.java:339) "RpcServer.default.FPBQ.Fifo.handler=8,queue=8,port=16020" #95 daemon prio=5 os_prio=0 tid=0x00007fc11d7fe800 nid=0x73d4 waiting for monitor entry [0x00007fc117304000] java.lang.Thread.State: BLOCKED (on object monitor) at org.apache.hadoop.fs.FSInputStream.read(FSInputStream.java:73) - waiting to lock <0x00007fc4c2769660> (a org.apache.hadoop.fs.s3a.S3AInputStream) at org.apache.hadoop.fs.FSDataInputStream.read(FSDataInputStream.java:97) at org.apache.hadoop.hbase.io.FileLink$FileLinkInputStream.read(FileLink.java:171) at org.apache.hadoop.fs.FSDataInputStream.read(FSDataInputStream.java:97) at org.apache.hadoop.hbase.io.util.BlockIOUtils.preadWithExtra(BlockIOUtils.java:235) at org.apache.hadoop.hbase.io.hfile.HFileBlock$FSReaderImpl.readAtOffset(HFileBlock.java:1564) at org.apache.hadoop.hbase.io.hfile.HFileBlock$FSReaderImpl.readBlockDataInternal(HFileBlock.java:1788) at org.apache.hadoop.hbase.io.hfile.HFileBlock$FSReaderImpl.readBlockData(HFileBlock.java:1597) at org.apache.hadoop.hbase.io.hfile.HFileReaderImpl.readBlock(HFileReaderImpl.java:1347) at org.apache.hadoop.hbase.io.hfile.HFileReaderImpl.readBlock(HFileReaderImpl.java:1268) at org.apache.hadoop.hbase.io.hfile.HFileReaderImpl$HFileScannerImpl.readNextDataBlock(HFileReaderImpl.java:762) at org.apache.hadoop.hbase.io.hfile.HFileReaderImpl$EncodedScanner.next(HFileReaderImpl.java:1540) at org.apache.hadoop.hbase.regionserver.StoreFileScanner.reseekAtOrAfter(StoreFileScanner.java:351) at org.apache.hadoop.hbase.regionserver.StoreFileScanner.reseek(StoreFileScanner.java:244) at org.apache.hadoop.hbase.regionserver.NonLazyKeyValueScanner.doRealSeek(NonLazyKeyValueScanner.java:55) at org.apache.hadoop.hbase.regionserver.KeyValueHeap.generalizedSeek(KeyValueHeap.java:322) at org.apache.hadoop.hbase.regionserver.KeyValueHeap.reseek(KeyValueHeap.java:265) at org.apache.hadoop.hbase.regionserver.StoreScanner.reseek(StoreScanner.java:1095) at org.apache.hadoop.hbase.regionserver.StoreScanner.seekAsDirection(StoreScanner.java:1084) at org.apache.hadoop.hbase.regionserver.StoreScanner.seekOrSkipToNextColumn(StoreScanner.java:821) at org.apache.hadoop.hbase.regionserver.StoreScanner.next(StoreScanner.java:684) at org.apache.hadoop.hbase.regionserver.KeyValueHeap.next(KeyValueHeap.java:155) at org.apache.hadoop.hbase.regionserver.HRegion$RegionScannerImpl.populateResult(HRegion.java:7399) at org.apache.hadoop.hbase.regionserver.HRegion$RegionScannerImpl.nextInternal(HRegion.java:7567) at org.apache.hadoop.hbase.regionserver.HRegion$RegionScannerImpl.nextRaw(HRegion.java:7331) at org.apache.hadoop.hbase.regionserver.HRegion$RegionScannerImpl.nextRaw(HRegion.java:7317) at org.apache.phoenix.coprocessor.PagedRegionScanner.next(PagedRegionScanner.java:57) at org.apache.phoenix.coprocessor.PagedRegionScanner.nextRaw(PagedRegionScanner.java:96) at org.apache.phoenix.iterate.RegionScannerFactory$1.nextRaw(RegionScannerFactory.java:211) at org.apache.phoenix.coprocessor.DelegateRegionScanner.nextRaw(DelegateRegionScanner.java:77) at org.apache.phoenix.coprocessor.DelegateRegionScanner.nextRaw(DelegateRegionScanner.java:77) at org.apache.phoenix.coprocessor.BaseScannerRegionObserver$RegionScannerHolder.nextRaw(BaseScannerRegionObserver.java:308) at org.apache.hadoop.hbase.regionserver.RSRpcServices.scan(RSRpcServices.java:3373) - locked <0x00007fc85f06a728> (a org.apache.phoenix.coprocessor.BaseScannerRegionObserver$RegionScannerHolder) at org.apache.hadoop.hbase.regionserver.RSRpcServices.scan(RSRpcServices.java:3637) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45819) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:392) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:140) at org.apache.hadoop.hbase.ipc.RpcExecutor$Handler.run(RpcExecutor.java:359) at org.apache.hadoop.hbase.ipc.RpcExecutor$Handler.run(RpcExecutor.java:339) "RpcServer.default.FPBQ.Fifo.handler=7,queue=7,port=16020" #94 daemon prio=5 os_prio=0 tid=0x00007fc11d7fc000 nid=0x73d3 waiting for monitor entry [0x00007fc117405000] java.lang.Thread.State: BLOCKED (on object monitor) at org.apache.hadoop.fs.FSInputStream.read(FSInputStream.java:73) - waiting to lock <0x00007fc4c2769660> (a org.apache.hadoop.fs.s3a.S3AInputStream) at org.apache.hadoop.fs.FSDataInputStream.read(FSDataInputStream.java:97) at org.apache.hadoop.hbase.io.FileLink$FileLinkInputStream.read(FileLink.java:171) at org.apache.hadoop.fs.FSDataInputStream.read(FSDataInputStream.java:97) at org.apache.hadoop.hbase.io.util.BlockIOUtils.preadWithExtra(BlockIOUtils.java:235) at org.apache.hadoop.hbase.io.hfile.HFileBlock$FSReaderImpl.readAtOffset(HFileBlock.java:1564) at org.apache.hadoop.hbase.io.hfile.HFileBlock$FSReaderImpl.readBlockDataInternal(HFileBlock.java:1788) at org.apache.hadoop.hbase.io.hfile.HFileBlock$FSReaderImpl.readBlockData(HFileBlock.java:1597) at org.apache.hadoop.hbase.io.hfile.HFileReaderImpl.readBlock(HFileReaderImpl.java:1347) at org.apache.hadoop.hbase.io.hfile.HFileReaderImpl.readBlock(HFileReaderImpl.java:1268) at org.apache.hadoop.hbase.io.hfile.HFileReaderImpl$HFileScannerImpl.readNextDataBlock(HFileReaderImpl.java:762) at org.apache.hadoop.hbase.io.hfile.HFileReaderImpl$EncodedScanner.next(HFileReaderImpl.java:1540) at org.apache.hadoop.hbase.regionserver.StoreFileScanner.reseekAtOrAfter(StoreFileScanner.java:351) at org.apache.hadoop.hbase.regionserver.StoreFileScanner.reseek(StoreFileScanner.java:244) at org.apache.hadoop.hbase.regionserver.NonLazyKeyValueScanner.doRealSeek(NonLazyKeyValueScanner.java:55) at org.apache.hadoop.hbase.regionserver.KeyValueHeap.generalizedSeek(KeyValueHeap.java:322) at org.apache.hadoop.hbase.regionserver.KeyValueHeap.reseek(KeyValueHeap.java:265) at org.apache.hadoop.hbase.regionserver.StoreScanner.reseek(StoreScanner.java:1095) at org.apache.hadoop.hbase.regionserver.StoreScanner.seekAsDirection(StoreScanner.java:1084) at org.apache.hadoop.hbase.regionserver.StoreScanner.seekOrSkipToNextColumn(StoreScanner.java:821) at org.apache.hadoop.hbase.regionserver.StoreScanner.next(StoreScanner.java:684) at org.apache.hadoop.hbase.regionserver.KeyValueHeap.next(KeyValueHeap.java:155) at org.apache.hadoop.hbase.regionserver.HRegion$RegionScannerImpl.populateResult(HRegion.java:7399) at org.apache.hadoop.hbase.regionserver.HRegion$RegionScannerImpl.nextInternal(HRegion.java:7567) at org.apache.hadoop.hbase.regionserver.HRegion$RegionScannerImpl.nextRaw(HRegion.java:7331) at org.apache.hadoop.hbase.regionserver.HRegion$RegionScannerImpl.nextRaw(HRegion.java:7317) at org.apache.phoenix.coprocessor.PagedRegionScanner.next(PagedRegionScanner.java:57) at org.apache.phoenix.coprocessor.PagedRegionScanner.nextRaw(PagedRegionScanner.java:96) at org.apache.phoenix.iterate.RegionScannerFactory$1.nextRaw(RegionScannerFactory.java:211) at org.apache.phoenix.coprocessor.DelegateRegionScanner.nextRaw(DelegateRegionScanner.java:77) at org.apache.phoenix.coprocessor.DelegateRegionScanner.nextRaw(DelegateRegionScanner.java:77) at org.apache.phoenix.coprocessor.BaseScannerRegionObserver$RegionScannerHolder.nextRaw(BaseScannerRegionObserver.java:308) at org.apache.hadoop.hbase.regionserver.RSRpcServices.scan(RSRpcServices.java:3373) - locked <0x00007fc85f049bd0> (a org.apache.phoenix.coprocessor.BaseScannerRegionObserver$RegionScannerHolder) at org.apache.hadoop.hbase.regionserver.RSRpcServices.scan(RSRpcServices.java:3637) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45819) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:392) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:140) at org.apache.hadoop.hbase.ipc.RpcExecutor$Handler.run(RpcExecutor.java:359) at org.apache.hadoop.hbase.ipc.RpcExecutor$Handler.run(RpcExecutor.java:339) "RpcServer.default.FPBQ.Fifo.handler=6,queue=6,port=16020" #93 daemon prio=5 os_prio=0 tid=0x00007fc11d7fa000 nid=0x73d2 waiting for monitor entry [0x00007fc117506000] java.lang.Thread.State: BLOCKED (on object monitor) at org.apache.hadoop.fs.FSInputStream.read(FSInputStream.java:73) - waiting to lock <0x00007fc4c2769660> (a org.apache.hadoop.fs.s3a.S3AInputStream) at org.apache.hadoop.fs.FSDataInputStream.read(FSDataInputStream.java:97) at org.apache.hadoop.hbase.io.FileLink$FileLinkInputStream.read(FileLink.java:171) at org.apache.hadoop.fs.FSDataInputStream.read(FSDataInputStream.java:97) at org.apache.hadoop.hbase.io.util.BlockIOUtils.preadWithExtra(BlockIOUtils.java:235) at org.apache.hadoop.hbase.io.hfile.HFileBlock$FSReaderImpl.readAtOffset(HFileBlock.java:1564) at org.apache.hadoop.hbase.io.hfile.HFileBlock$FSReaderImpl.readBlockDataInternal(HFileBlock.java:1788) at org.apache.hadoop.hbase.io.hfile.HFileBlock$FSReaderImpl.readBlockData(HFileBlock.java:1597) at org.apache.hadoop.hbase.io.hfile.HFileReaderImpl.readBlock(HFileReaderImpl.java:1347) at org.apache.hadoop.hbase.io.hfile.HFileReaderImpl.readBlock(HFileReaderImpl.java:1268) at org.apache.hadoop.hbase.io.hfile.HFileReaderImpl$HFileScannerImpl.readNextDataBlock(HFileReaderImpl.java:762) at org.apache.hadoop.hbase.io.hfile.HFileReaderImpl$EncodedScanner.next(HFileReaderImpl.java:1540) at org.apache.hadoop.hbase.regionserver.StoreFileScanner.reseekAtOrAfter(StoreFileScanner.java:351) at org.apache.hadoop.hbase.regionserver.StoreFileScanner.reseek(StoreFileScanner.java:244) at org.apache.hadoop.hbase.regionserver.NonLazyKeyValueScanner.doRealSeek(NonLazyKeyValueScanner.java:55) at org.apache.hadoop.hbase.regionserver.KeyValueHeap.generalizedSeek(KeyValueHeap.java:322) at org.apache.hadoop.hbase.regionserver.KeyValueHeap.reseek(KeyValueHeap.java:265) at org.apache.hadoop.hbase.regionserver.StoreScanner.reseek(StoreScanner.java:1095) at org.apache.hadoop.hbase.regionserver.StoreScanner.seekAsDirection(StoreScanner.java:1084) at org.apache.hadoop.hbase.regionserver.StoreScanner.seekOrSkipToNextColumn(StoreScanner.java:821) at org.apache.hadoop.hbase.regionserver.StoreScanner.next(StoreScanner.java:684) at org.apache.hadoop.hbase.regionserver.KeyValueHeap.next(KeyValueHeap.java:155) at org.apache.hadoop.hbase.regionserver.HRegion$RegionScannerImpl.populateResult(HRegion.java:7399) at org.apache.hadoop.hbase.regionserver.HRegion$RegionScannerImpl.nextInternal(HRegion.java:7567) at org.apache.hadoop.hbase.regionserver.HRegion$RegionScannerImpl.nextRaw(HRegion.java:7331) at org.apache.hadoop.hbase.regionserver.HRegion$RegionScannerImpl.nextRaw(HRegion.java:7317) at org.apache.phoenix.coprocessor.PagedRegionScanner.next(PagedRegionScanner.java:57) at org.apache.phoenix.coprocessor.PagedRegionScanner.nextRaw(PagedRegionScanner.java:96) at org.apache.phoenix.iterate.RegionScannerFactory$1.nextRaw(RegionScannerFactory.java:211) at org.apache.phoenix.coprocessor.DelegateRegionScanner.nextRaw(DelegateRegionScanner.java:77) at org.apache.phoenix.coprocessor.DelegateRegionScanner.nextRaw(DelegateRegionScanner.java:77) at org.apache.phoenix.coprocessor.BaseScannerRegionObserver$RegionScannerHolder.nextRaw(BaseScannerRegionObserver.java:308) at org.apache.hadoop.hbase.regionserver.RSRpcServices.scan(RSRpcServices.java:3373) - locked <0x00007fc6b13a3540> (a org.apache.phoenix.coprocessor.BaseScannerRegionObserver$RegionScannerHolder) at org.apache.hadoop.hbase.regionserver.RSRpcServices.scan(RSRpcServices.java:3637) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45819) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:392) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:140) at org.apache.hadoop.hbase.ipc.RpcExecutor$Handler.run(RpcExecutor.java:359) at org.apache.hadoop.hbase.ipc.RpcExecutor$Handler.run(RpcExecutor.java:339) "RpcServer.default.FPBQ.Fifo.handler=5,queue=5,port=16020" #92 daemon prio=5 os_prio=0 tid=0x00007fc11d7f8000 nid=0x73d1 waiting for monitor entry [0x00007fc117607000] java.lang.Thread.State: BLOCKED (on object monitor) at org.apache.hadoop.fs.FSInputStream.read(FSInputStream.java:73) - waiting to lock <0x00007fc4c2769660> (a org.apache.hadoop.fs.s3a.S3AInputStream) at org.apache.hadoop.fs.FSDataInputStream.read(FSDataInputStream.java:97) at org.apache.hadoop.hbase.io.FileLink$FileLinkInputStream.read(FileLink.java:171) at org.apache.hadoop.fs.FSDataInputStream.read(FSDataInputStream.java:97) at org.apache.hadoop.hbase.io.util.BlockIOUtils.preadWithExtra(BlockIOUtils.java:235) at org.apache.hadoop.hbase.io.hfile.HFileBlock$FSReaderImpl.readAtOffset(HFileBlock.java:1564) at org.apache.hadoop.hbase.io.hfile.HFileBlock$FSReaderImpl.readBlockDataInternal(HFileBlock.java:1788) at org.apache.hadoop.hbase.io.hfile.HFileBlock$FSReaderImpl.readBlockData(HFileBlock.java:1597) at org.apache.hadoop.hbase.io.hfile.HFileReaderImpl.readBlock(HFileReaderImpl.java:1347) at org.apache.hadoop.hbase.io.hfile.HFileReaderImpl.readBlock(HFileReaderImpl.java:1268) at org.apache.hadoop.hbase.io.hfile.HFileReaderImpl$HFileScannerImpl.readNextDataBlock(HFileReaderImpl.java:762) at org.apache.hadoop.hbase.io.hfile.HFileReaderImpl$EncodedScanner.next(HFileReaderImpl.java:1540) at org.apache.hadoop.hbase.regionserver.StoreFileScanner.reseekAtOrAfter(StoreFileScanner.java:351) at org.apache.hadoop.hbase.regionserver.StoreFileScanner.reseek(StoreFileScanner.java:244) at org.apache.hadoop.hbase.regionserver.NonLazyKeyValueScanner.doRealSeek(NonLazyKeyValueScanner.java:55) at org.apache.hadoop.hbase.regionserver.KeyValueHeap.generalizedSeek(KeyValueHeap.java:322) at org.apache.hadoop.hbase.regionserver.KeyValueHeap.reseek(KeyValueHeap.java:265) at org.apache.hadoop.hbase.regionserver.StoreScanner.reseek(StoreScanner.java:1095) at org.apache.hadoop.hbase.regionserver.StoreScanner.seekAsDirection(StoreScanner.java:1084) at org.apache.hadoop.hbase.regionserver.StoreScanner.seekOrSkipToNextColumn(StoreScanner.java:821) at org.apache.hadoop.hbase.regionserver.StoreScanner.next(StoreScanner.java:684) at org.apache.hadoop.hbase.regionserver.KeyValueHeap.next(KeyValueHeap.java:155) at org.apache.hadoop.hbase.regionserver.HRegion$RegionScannerImpl.populateResult(HRegion.java:7399) at org.apache.hadoop.hbase.regionserver.HRegion$RegionScannerImpl.nextInternal(HRegion.java:7567) at org.apache.hadoop.hbase.regionserver.HRegion$RegionScannerImpl.nextRaw(HRegion.java:7331) at org.apache.hadoop.hbase.regionserver.HRegion$RegionScannerImpl.nextRaw(HRegion.java:7317) at org.apache.phoenix.coprocessor.PagedRegionScanner.next(PagedRegionScanner.java:57) at org.apache.phoenix.coprocessor.PagedRegionScanner.nextRaw(PagedRegionScanner.java:96) at org.apache.phoenix.iterate.RegionScannerFactory$1.nextRaw(RegionScannerFactory.java:211) at org.apache.phoenix.coprocessor.DelegateRegionScanner.nextRaw(DelegateRegionScanner.java:77) at org.apache.phoenix.coprocessor.DelegateRegionScanner.nextRaw(DelegateRegionScanner.java:77) at org.apache.phoenix.coprocessor.BaseScannerRegionObserver$RegionScannerHolder.nextRaw(BaseScannerRegionObserver.java:308) at org.apache.hadoop.hbase.regionserver.RSRpcServices.scan(RSRpcServices.java:3373) - locked <0x00007fc85f064e00> (a org.apache.phoenix.coprocessor.BaseScannerRegionObserver$RegionScannerHolder) at org.apache.hadoop.hbase.regionserver.RSRpcServices.scan(RSRpcServices.java:3637) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45819) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:392) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:140) at org.apache.hadoop.hbase.ipc.RpcExecutor$Handler.run(RpcExecutor.java:359) at org.apache.hadoop.hbase.ipc.RpcExecutor$Handler.run(RpcExecutor.java:339) "RpcServer.default.FPBQ.Fifo.handler=4,queue=4,port=16020" #91 daemon prio=5 os_prio=0 tid=0x00007fc11d7f5800 nid=0x73d0 waiting for monitor entry [0x00007fc117708000] java.lang.Thread.State: BLOCKED (on object monitor) at org.apache.hadoop.fs.FSInputStream.read(FSInputStream.java:73) - waiting to lock <0x00007fc4c2769660> (a org.apache.hadoop.fs.s3a.S3AInputStream) at org.apache.hadoop.fs.FSDataInputStream.read(FSDataInputStream.java:97) at org.apache.hadoop.hbase.io.FileLink$FileLinkInputStream.read(FileLink.java:171) at org.apache.hadoop.fs.FSDataInputStream.read(FSDataInputStream.java:97) at org.apache.hadoop.hbase.io.util.BlockIOUtils.preadWithExtra(BlockIOUtils.java:235) at org.apache.hadoop.hbase.io.hfile.HFileBlock$FSReaderImpl.readAtOffset(HFileBlock.java:1564) at org.apache.hadoop.hbase.io.hfile.HFileBlock$FSReaderImpl.readBlockDataInternal(HFileBlock.java:1788) at org.apache.hadoop.hbase.io.hfile.HFileBlock$FSReaderImpl.readBlockData(HFileBlock.java:1597) at org.apache.hadoop.hbase.io.hfile.HFileReaderImpl.readBlock(HFileReaderImpl.java:1347) at org.apache.hadoop.hbase.io.hfile.HFileReaderImpl.readBlock(HFileReaderImpl.java:1268) at org.apache.hadoop.hbase.io.hfile.HFileReaderImpl$HFileScannerImpl.readNextDataBlock(HFileReaderImpl.java:762) at org.apache.hadoop.hbase.io.hfile.HFileReaderImpl$EncodedScanner.next(HFileReaderImpl.java:1540) at org.apache.hadoop.hbase.regionserver.StoreFileScanner.reseekAtOrAfter(StoreFileScanner.java:351) at org.apache.hadoop.hbase.regionserver.StoreFileScanner.reseek(StoreFileScanner.java:244) at org.apache.hadoop.hbase.regionserver.NonLazyKeyValueScanner.doRealSeek(NonLazyKeyValueScanner.java:55) at org.apache.hadoop.hbase.regionserver.KeyValueHeap.generalizedSeek(KeyValueHeap.java:322) at org.apache.hadoop.hbase.regionserver.KeyValueHeap.reseek(KeyValueHeap.java:265) at org.apache.hadoop.hbase.regionserver.StoreScanner.reseek(StoreScanner.java:1095) at org.apache.hadoop.hbase.regionserver.StoreScanner.seekAsDirection(StoreScanner.java:1084) at org.apache.hadoop.hbase.regionserver.StoreScanner.seekOrSkipToNextColumn(StoreScanner.java:821) at org.apache.hadoop.hbase.regionserver.StoreScanner.next(StoreScanner.java:684) at org.apache.hadoop.hbase.regionserver.KeyValueHeap.next(KeyValueHeap.java:155) at org.apache.hadoop.hbase.regionserver.HRegion$RegionScannerImpl.populateResult(HRegion.java:7399) at org.apache.hadoop.hbase.regionserver.HRegion$RegionScannerImpl.nextInternal(HRegion.java:7567) at org.apache.hadoop.hbase.regionserver.HRegion$RegionScannerImpl.nextRaw(HRegion.java:7331) at org.apache.hadoop.hbase.regionserver.HRegion$RegionScannerImpl.nextRaw(HRegion.java:7317) at org.apache.phoenix.coprocessor.PagedRegionScanner.next(PagedRegionScanner.java:57) at org.apache.phoenix.coprocessor.PagedRegionScanner.nextRaw(PagedRegionScanner.java:96) at org.apache.phoenix.iterate.RegionScannerFactory$1.nextRaw(RegionScannerFactory.java:211) at org.apache.phoenix.coprocessor.DelegateRegionScanner.nextRaw(DelegateRegionScanner.java:77) at org.apache.phoenix.coprocessor.DelegateRegionScanner.nextRaw(DelegateRegionScanner.java:77) at org.apache.phoenix.coprocessor.BaseScannerRegionObserver$RegionScannerHolder.nextRaw(BaseScannerRegionObserver.java:308) at org.apache.hadoop.hbase.regionserver.RSRpcServices.scan(RSRpcServices.java:3373) - locked <0x00007fc85f0bd4e0> (a org.apache.phoenix.coprocessor.BaseScannerRegionObserver$RegionScannerHolder) at org.apache.hadoop.hbase.regionserver.RSRpcServices.scan(RSRpcServices.java:3637) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45819) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:392) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:140) at org.apache.hadoop.hbase.ipc.RpcExecutor$Handler.run(RpcExecutor.java:359) at org.apache.hadoop.hbase.ipc.RpcExecutor$Handler.run(RpcExecutor.java:339) "RpcServer.default.FPBQ.Fifo.handler=3,queue=3,port=16020" #90 daemon prio=5 os_prio=0 tid=0x00007fc11d7f3800 nid=0x73cf waiting for monitor entry [0x00007fc117809000] java.lang.Thread.State: BLOCKED (on object monitor) at org.apache.hadoop.fs.FSInputStream.read(FSInputStream.java:73) - waiting to lock <0x00007fc4c2769660> (a org.apache.hadoop.fs.s3a.S3AInputStream) at org.apache.hadoop.fs.FSDataInputStream.read(FSDataInputStream.java:97) at org.apache.hadoop.hbase.io.FileLink$FileLinkInputStream.read(FileLink.java:171) at org.apache.hadoop.fs.FSDataInputStream.read(FSDataInputStream.java:97) at org.apache.hadoop.hbase.io.util.BlockIOUtils.preadWithExtra(BlockIOUtils.java:235) at org.apache.hadoop.hbase.io.hfile.HFileBlock$FSReaderImpl.readAtOffset(HFileBlock.java:1564) at org.apache.hadoop.hbase.io.hfile.HFileBlock$FSReaderImpl.readBlockDataInternal(HFileBlock.java:1788) at org.apache.hadoop.hbase.io.hfile.HFileBlock$FSReaderImpl.readBlockData(HFileBlock.java:1597) at org.apache.hadoop.hbase.io.hfile.HFileReaderImpl.readBlock(HFileReaderImpl.java:1347) at org.apache.hadoop.hbase.io.hfile.HFileReaderImpl.readBlock(HFileReaderImpl.java:1268) at org.apache.hadoop.hbase.io.hfile.HFileReaderImpl$HFileScannerImpl.readNextDataBlock(HFileReaderImpl.java:762) at org.apache.hadoop.hbase.io.hfile.HFileReaderImpl$EncodedScanner.next(HFileReaderImpl.java:1540) at org.apache.hadoop.hbase.regionserver.StoreFileScanner.reseekAtOrAfter(StoreFileScanner.java:351) at org.apache.hadoop.hbase.regionserver.StoreFileScanner.reseek(StoreFileScanner.java:244) at org.apache.hadoop.hbase.regionserver.NonLazyKeyValueScanner.doRealSeek(NonLazyKeyValueScanner.java:55) at org.apache.hadoop.hbase.regionserver.KeyValueHeap.generalizedSeek(KeyValueHeap.java:322) at org.apache.hadoop.hbase.regionserver.KeyValueHeap.reseek(KeyValueHeap.java:265) at org.apache.hadoop.hbase.regionserver.StoreScanner.reseek(StoreScanner.java:1095) at org.apache.hadoop.hbase.regionserver.StoreScanner.seekAsDirection(StoreScanner.java:1084) at org.apache.hadoop.hbase.regionserver.StoreScanner.seekOrSkipToNextColumn(StoreScanner.java:821) at org.apache.hadoop.hbase.regionserver.StoreScanner.next(StoreScanner.java:684) at org.apache.hadoop.hbase.regionserver.KeyValueHeap.next(KeyValueHeap.java:155) at org.apache.hadoop.hbase.regionserver.HRegion$RegionScannerImpl.populateResult(HRegion.java:7399) at org.apache.hadoop.hbase.regionserver.HRegion$RegionScannerImpl.nextInternal(HRegion.java:7567) at org.apache.hadoop.hbase.regionserver.HRegion$RegionScannerImpl.nextRaw(HRegion.java:7331) at org.apache.hadoop.hbase.regionserver.HRegion$RegionScannerImpl.nextRaw(HRegion.java:7317) at org.apache.phoenix.coprocessor.PagedRegionScanner.next(PagedRegionScanner.java:57) at org.apache.phoenix.coprocessor.PagedRegionScanner.nextRaw(PagedRegionScanner.java:96) at org.apache.phoenix.iterate.RegionScannerFactory$1.nextRaw(RegionScannerFactory.java:211) at org.apache.phoenix.coprocessor.DelegateRegionScanner.nextRaw(DelegateRegionScanner.java:77) at org.apache.phoenix.coprocessor.DelegateRegionScanner.nextRaw(DelegateRegionScanner.java:77) at org.apache.phoenix.coprocessor.BaseScannerRegionObserver$RegionScannerHolder.nextRaw(BaseScannerRegionObserver.java:308) at org.apache.hadoop.hbase.regionserver.RSRpcServices.scan(RSRpcServices.java:3373) - locked <0x00007fc85f04a1d8> (a org.apache.phoenix.coprocessor.BaseScannerRegionObserver$RegionScannerHolder) at org.apache.hadoop.hbase.regionserver.RSRpcServices.scan(RSRpcServices.java:3637) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45819) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:392) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:140) at org.apache.hadoop.hbase.ipc.RpcExecutor$Handler.run(RpcExecutor.java:359) at org.apache.hadoop.hbase.ipc.RpcExecutor$Handler.run(RpcExecutor.java:339) "RpcServer.default.FPBQ.Fifo.handler=2,queue=2,port=16020" #89 daemon prio=5 os_prio=0 tid=0x00007fc11d7f2000 nid=0x73ce waiting for monitor entry [0x00007fc11790a000] java.lang.Thread.State: BLOCKED (on object monitor) at org.apache.hadoop.fs.FSInputStream.read(FSInputStream.java:73) - waiting to lock <0x00007fc4c2769660> (a org.apache.hadoop.fs.s3a.S3AInputStream) at org.apache.hadoop.fs.FSDataInputStream.read(FSDataInputStream.java:97) at org.apache.hadoop.hbase.io.FileLink$FileLinkInputStream.read(FileLink.java:171) at org.apache.hadoop.fs.FSDataInputStream.read(FSDataInputStream.java:97) at org.apache.hadoop.hbase.io.util.BlockIOUtils.preadWithExtra(BlockIOUtils.java:235) at org.apache.hadoop.hbase.io.hfile.HFileBlock$FSReaderImpl.readAtOffset(HFileBlock.java:1564) at org.apache.hadoop.hbase.io.hfile.HFileBlock$FSReaderImpl.readBlockDataInternal(HFileBlock.java:1788) at org.apache.hadoop.hbase.io.hfile.HFileBlock$FSReaderImpl.readBlockData(HFileBlock.java:1597) at org.apache.hadoop.hbase.io.hfile.HFileReaderImpl.readBlock(HFileReaderImpl.java:1347) at org.apache.hadoop.hbase.io.hfile.HFileReaderImpl.readBlock(HFileReaderImpl.java:1268) at org.apache.hadoop.hbase.io.hfile.HFileReaderImpl$HFileScannerImpl.readNextDataBlock(HFileReaderImpl.java:762) at org.apache.hadoop.hbase.io.hfile.HFileReaderImpl$EncodedScanner.next(HFileReaderImpl.java:1540) at org.apache.hadoop.hbase.regionserver.StoreFileScanner.reseekAtOrAfter(StoreFileScanner.java:351) at org.apache.hadoop.hbase.regionserver.StoreFileScanner.reseek(StoreFileScanner.java:244) at org.apache.hadoop.hbase.regionserver.NonLazyKeyValueScanner.doRealSeek(NonLazyKeyValueScanner.java:55) at org.apache.hadoop.hbase.regionserver.KeyValueHeap.generalizedSeek(KeyValueHeap.java:322) at org.apache.hadoop.hbase.regionserver.KeyValueHeap.reseek(KeyValueHeap.java:265) at org.apache.hadoop.hbase.regionserver.StoreScanner.reseek(StoreScanner.java:1095) at org.apache.hadoop.hbase.regionserver.StoreScanner.seekAsDirection(StoreScanner.java:1084) at org.apache.hadoop.hbase.regionserver.StoreScanner.seekOrSkipToNextColumn(StoreScanner.java:821) at org.apache.hadoop.hbase.regionserver.StoreScanner.next(StoreScanner.java:684) at org.apache.hadoop.hbase.regionserver.KeyValueHeap.next(KeyValueHeap.java:155) at org.apache.hadoop.hbase.regionserver.HRegion$RegionScannerImpl.populateResult(HRegion.java:7399) at org.apache.hadoop.hbase.regionserver.HRegion$RegionScannerImpl.nextInternal(HRegion.java:7567) at org.apache.hadoop.hbase.regionserver.HRegion$RegionScannerImpl.nextRaw(HRegion.java:7331) at org.apache.hadoop.hbase.regionserver.HRegion$RegionScannerImpl.nextRaw(HRegion.java:7317) at org.apache.phoenix.coprocessor.PagedRegionScanner.next(PagedRegionScanner.java:57) at org.apache.phoenix.coprocessor.PagedRegionScanner.nextRaw(PagedRegionScanner.java:96) at org.apache.phoenix.iterate.RegionScannerFactory$1.nextRaw(RegionScannerFactory.java:211) at org.apache.phoenix.coprocessor.DelegateRegionScanner.nextRaw(DelegateRegionScanner.java:77) at org.apache.phoenix.coprocessor.DelegateRegionScanner.nextRaw(DelegateRegionScanner.java:77) at org.apache.phoenix.coprocessor.BaseScannerRegionObserver$RegionScannerHolder.nextRaw(BaseScannerRegionObserver.java:308) at org.apache.hadoop.hbase.regionserver.RSRpcServices.scan(RSRpcServices.java:3373) - locked <0x00007fc9723ce788> (a org.apache.phoenix.coprocessor.BaseScannerRegionObserver$RegionScannerHolder) at org.apache.hadoop.hbase.regionserver.RSRpcServices.scan(RSRpcServices.java:3637) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45819) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:392) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:140) at org.apache.hadoop.hbase.ipc.RpcExecutor$Handler.run(RpcExecutor.java:359) at org.apache.hadoop.hbase.ipc.RpcExecutor$Handler.run(RpcExecutor.java:339) "RpcServer.default.FPBQ.Fifo.handler=1,queue=1,port=16020" #88 daemon prio=5 os_prio=0 tid=0x00007fc11d7f0000 nid=0x73cd waiting on condition [0x00007fc117a0c000] java.lang.Thread.State: WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007fc1c9fedba0> (a java.util.concurrent.Semaphore$NonfairSync) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer.parkAndCheckInterrupt(AbstractQueuedSynchronizer.java:836) at java.util.concurrent.locks.AbstractQueuedSynchronizer.doAcquireSharedInterruptibly(AbstractQueuedSynchronizer.java:997) at java.util.concurrent.locks.AbstractQueuedSynchronizer.acquireSharedInterruptibly(AbstractQueuedSynchronizer.java:1304) at java.util.concurrent.Semaphore.acquire(Semaphore.java:312) at org.apache.hadoop.hbase.ipc.FastPathBalancedQueueRpcExecutor$FastPathHandler.getCallRunner(FastPathBalancedQueueRpcExecutor.java:109) at org.apache.hadoop.hbase.ipc.RpcExecutor$Handler.run(RpcExecutor.java:339) "RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=16020" #87 daemon prio=5 os_prio=0 tid=0x00007fc11d7ee800 nid=0x73cc waiting for monitor entry [0x00007fc117b0c000] java.lang.Thread.State: BLOCKED (on object monitor) at org.apache.hadoop.fs.FSInputStream.read(FSInputStream.java:73) - waiting to lock <0x00007fc4c2769660> (a org.apache.hadoop.fs.s3a.S3AInputStream) at org.apache.hadoop.fs.FSDataInputStream.read(FSDataInputStream.java:97) at org.apache.hadoop.hbase.io.FileLink$FileLinkInputStream.read(FileLink.java:171) at org.apache.hadoop.fs.FSDataInputStream.read(FSDataInputStream.java:97) at org.apache.hadoop.hbase.io.util.BlockIOUtils.preadWithExtra(BlockIOUtils.java:235) at org.apache.hadoop.hbase.io.hfile.HFileBlock$FSReaderImpl.readAtOffset(HFileBlock.java:1564) at org.apache.hadoop.hbase.io.hfile.HFileBlock$FSReaderImpl.readBlockDataInternal(HFileBlock.java:1788) at org.apache.hadoop.hbase.io.hfile.HFileBlock$FSReaderImpl.readBlockData(HFileBlock.java:1597) at org.apache.hadoop.hbase.io.hfile.HFileReaderImpl.readBlock(HFileReaderImpl.java:1347) at org.apache.hadoop.hbase.io.hfile.HFileReaderImpl.readBlock(HFileReaderImpl.java:1268) at org.apache.hadoop.hbase.io.hfile.HFileReaderImpl$HFileScannerImpl.readNextDataBlock(HFileReaderImpl.java:762) at org.apache.hadoop.hbase.io.hfile.HFileReaderImpl$EncodedScanner.next(HFileReaderImpl.java:1540) at org.apache.hadoop.hbase.regionserver.StoreFileScanner.reseekAtOrAfter(StoreFileScanner.java:351) at org.apache.hadoop.hbase.regionserver.StoreFileScanner.reseek(StoreFileScanner.java:244) at org.apache.hadoop.hbase.regionserver.NonLazyKeyValueScanner.doRealSeek(NonLazyKeyValueScanner.java:55) at org.apache.hadoop.hbase.regionserver.KeyValueHeap.generalizedSeek(KeyValueHeap.java:322) at org.apache.hadoop.hbase.regionserver.KeyValueHeap.reseek(KeyValueHeap.java:265) at org.apache.hadoop.hbase.regionserver.StoreScanner.reseek(StoreScanner.java:1095) at org.apache.hadoop.hbase.regionserver.StoreScanner.seekAsDirection(StoreScanner.java:1084) at org.apache.hadoop.hbase.regionserver.StoreScanner.seekOrSkipToNextColumn(StoreScanner.java:821) at org.apache.hadoop.hbase.regionserver.StoreScanner.next(StoreScanner.java:684) at org.apache.hadoop.hbase.regionserver.KeyValueHeap.next(KeyValueHeap.java:155) at org.apache.hadoop.hbase.regionserver.HRegion$RegionScannerImpl.populateResult(HRegion.java:7399) at org.apache.hadoop.hbase.regionserver.HRegion$RegionScannerImpl.nextInternal(HRegion.java:7567) at org.apache.hadoop.hbase.regionserver.HRegion$RegionScannerImpl.nextRaw(HRegion.java:7331) at org.apache.hadoop.hbase.regionserver.HRegion$RegionScannerImpl.nextRaw(HRegion.java:7317) at org.apache.phoenix.coprocessor.PagedRegionScanner.next(PagedRegionScanner.java:57) at org.apache.phoenix.coprocessor.PagedRegionScanner.nextRaw(PagedRegionScanner.java:96) at org.apache.phoenix.iterate.RegionScannerFactory$1.nextRaw(RegionScannerFactory.java:211) at org.apache.phoenix.coprocessor.DelegateRegionScanner.nextRaw(DelegateRegionScanner.java:77) at org.apache.phoenix.coprocessor.DelegateRegionScanner.nextRaw(DelegateRegionScanner.java:77) at org.apache.phoenix.coprocessor.BaseScannerRegionObserver$RegionScannerHolder.nextRaw(BaseScannerRegionObserver.java:308) at org.apache.hadoop.hbase.regionserver.RSRpcServices.scan(RSRpcServices.java:3373) - locked <0x00007fc85f06c858> (a org.apache.phoenix.coprocessor.BaseScannerRegionObserver$RegionScannerHolder) at org.apache.hadoop.hbase.regionserver.RSRpcServices.scan(RSRpcServices.java:3637) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45819) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:392) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:140) at org.apache.hadoop.hbase.ipc.RpcExecutor$Handler.run(RpcExecutor.java:359) at org.apache.hadoop.hbase.ipc.RpcExecutor$Handler.run(RpcExecutor.java:339) "ZKSecretWatcher-leaderElector" #86 daemon prio=5 os_prio=0 tid=0x00007fc11d7e6800 nid=0x73cb in Object.wait() [0x00007fc117c0e000] java.lang.Thread.State: WAITING (on object monitor) at java.lang.Object.wait(Native Method) - waiting on <0x00007fc1c9ff5d98> (a java.lang.Object) at java.lang.Object.wait(Object.java:502) at org.apache.hadoop.hbase.zookeeper.ZKLeaderManager.waitToBecomeLeader(ZKLeaderManager.java:143) - locked <0x00007fc1c9ff5d98> (a java.lang.Object) at org.apache.hadoop.hbase.security.token.AuthenticationTokenSecretManager$LeaderElector.run(AuthenticationTokenSecretManager.java:328) "zk-event-processor-pool-0" #85 daemon prio=5 os_prio=0 tid=0x0000558b13d61800 nid=0x73ca waiting on condition [0x00007fc117d0f000] java.lang.Thread.State: WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007fc1c9fedf98> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:750) "main-EventThread" #83 daemon prio=5 os_prio=0 tid=0x00007fc11d796000 nid=0x73c7 waiting on condition [0x00007fc117f11000] java.lang.Thread.State: WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007fc1c9ff5f60> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) at org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:503) "main-SendThread(cod-1e3xsc06g8mjm-master0.exait3p.z1did1.b0.cloudera.site:2181)" #82 daemon prio=5 os_prio=0 tid=0x00007fc11d795000 nid=0x73c6 runnable [0x00007fc118012000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007fc1c9fee4b8> (a sun.nio.ch.Util$3) - locked <0x00007fc1c9fee4c8> (a java.util.Collections$UnmodifiableSet) - locked <0x00007fc1c9fee470> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.zookeeper.ClientCnxnSocketNIO.doTransport(ClientCnxnSocketNIO.java:345) at org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1225) "java-sdk-http-connection-reaper" #81 daemon prio=5 os_prio=0 tid=0x00007fc11d711000 nid=0x73c5 waiting on condition [0x00007fc118313000] java.lang.Thread.State: TIMED_WAITING (sleeping) at java.lang.Thread.sleep(Native Method) at com.amazonaws.http.IdleConnectionReaper.run(IdleConnectionReaper.java:188) "Timer for 's3a-file-system' metrics system" #80 daemon prio=5 os_prio=0 tid=0x00007fc11d175000 nid=0x73c2 in Object.wait() [0x00007fc118e14000] java.lang.Thread.State: TIMED_WAITING (on object monitor) at java.lang.Object.wait(Native Method) at java.util.TimerThread.mainLoop(Timer.java:552) - locked <0x00007fc1c9ff6148> (a java.util.TaskQueue) at java.util.TimerThread.run(Timer.java:505) "MutableQuantiles-0" #79 daemon prio=5 os_prio=0 tid=0x00007fc11d172000 nid=0x73c1 waiting on condition [0x00007fc118f15000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007fc1c9ff6310> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1093) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:750) "client DomainSocketWatcher" #76 daemon prio=5 os_prio=0 tid=0x00007fc99aca9000 nid=0x73bf runnable [0x00007fc119016000] java.lang.Thread.State: RUNNABLE at org.apache.hadoop.net.unix.DomainSocketWatcher.doPoll0(Native Method) at org.apache.hadoop.net.unix.DomainSocketWatcher.access$900(DomainSocketWatcher.java:52) at org.apache.hadoop.net.unix.DomainSocketWatcher$2.run(DomainSocketWatcher.java:503) at java.lang.Thread.run(Thread.java:750) "org.apache.hadoop.fs.FileSystem$Statistics$StatisticsDataReferenceCleaner" #75 daemon prio=5 os_prio=0 tid=0x00007fc99ab5b800 nid=0x73bd in Object.wait() [0x00007fc1199a5000] java.lang.Thread.State: WAITING (on object monitor) at java.lang.Object.wait(Native Method) at java.lang.ref.ReferenceQueue.remove(ReferenceQueue.java:144) - locked <0x00007fc1bc116a08> (a java.lang.ref.ReferenceQueue$Lock) at java.lang.ref.ReferenceQueue.remove(ReferenceQueue.java:165) at org.apache.hadoop.fs.FileSystem$Statistics$StatisticsDataReferenceCleaner.run(FileSystem.java:4003) at java.lang.Thread.run(Thread.java:750) "MobFileCache #0" #74 daemon prio=5 os_prio=0 tid=0x00007fc99b917800 nid=0x73bc waiting on condition [0x00007fc119ca6000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007fc1bc116bc0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1093) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:750) "BucketCacheStatsExecutor" #72 daemon prio=5 os_prio=0 tid=0x00007fc11c4af800 nid=0x73bb waiting on condition [0x00007fc119da7000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007fc1bc116e20> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1093) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:750) "main-BucketCacheWriter-7" #70 daemon prio=5 os_prio=0 tid=0x00007fc11c4ae800 nid=0x73ba waiting on condition [0x00007fc119ea8000] java.lang.Thread.State: WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007fc1bc117090> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) at java.util.concurrent.ArrayBlockingQueue.take(ArrayBlockingQueue.java:403) at org.apache.hadoop.hbase.io.hfile.bucket.BucketCache.getRAMQueueEntries(BucketCache.java:1168) at org.apache.hadoop.hbase.io.hfile.bucket.BucketCache$WriterThread.run(BucketCache.java:984) "main-BucketCacheWriter-6" #69 daemon prio=5 os_prio=0 tid=0x00007fc11c4ae000 nid=0x73b9 runnable [0x00007fc119fa9000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.FileDispatcherImpl.force0(Native Method) at sun.nio.ch.FileDispatcherImpl.force(FileDispatcherImpl.java:80) at sun.nio.ch.FileChannelImpl.force(FileChannelImpl.java:388) at org.apache.hadoop.hbase.io.hfile.bucket.FileIOEngine.sync(FileIOEngine.java:185) at org.apache.hadoop.hbase.io.hfile.bucket.BucketCache.doDrain(BucketCache.java:1109) at org.apache.hadoop.hbase.io.hfile.bucket.BucketCache$WriterThread.run(BucketCache.java:990) "main-BucketCacheWriter-5" #68 daemon prio=5 os_prio=0 tid=0x00007fc99b7b0800 nid=0x73b8 waiting on condition [0x00007fc11a0aa000] java.lang.Thread.State: WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007fc1bc1172a8> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) at java.util.concurrent.ArrayBlockingQueue.take(ArrayBlockingQueue.java:403) at org.apache.hadoop.hbase.io.hfile.bucket.BucketCache.getRAMQueueEntries(BucketCache.java:1168) at org.apache.hadoop.hbase.io.hfile.bucket.BucketCache$WriterThread.run(BucketCache.java:984) "main-BucketCacheWriter-4" #67 daemon prio=5 os_prio=0 tid=0x00007fc99b7b0000 nid=0x73b7 waiting on condition [0x00007fc11a1ab000] java.lang.Thread.State: WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007fc1bc095c88> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) at java.util.concurrent.ArrayBlockingQueue.take(ArrayBlockingQueue.java:403) at org.apache.hadoop.hbase.io.hfile.bucket.BucketCache.getRAMQueueEntries(BucketCache.java:1168) at org.apache.hadoop.hbase.io.hfile.bucket.BucketCache$WriterThread.run(BucketCache.java:984) "main-BucketCacheWriter-3" #66 daemon prio=5 os_prio=0 tid=0x00007fc11c47a000 nid=0x73b6 waiting on condition [0x00007fc11a2ac000] java.lang.Thread.State: WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007fc1bc1174c0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) at java.util.concurrent.ArrayBlockingQueue.take(ArrayBlockingQueue.java:403) at org.apache.hadoop.hbase.io.hfile.bucket.BucketCache.getRAMQueueEntries(BucketCache.java:1168) at org.apache.hadoop.hbase.io.hfile.bucket.BucketCache$WriterThread.run(BucketCache.java:984) "main-BucketCacheWriter-2" #65 daemon prio=5 os_prio=0 tid=0x00007fc11c479000 nid=0x73b5 waiting on condition [0x00007fc11a3ad000] java.lang.Thread.State: WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007fc1bc095ea0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) at java.util.concurrent.ArrayBlockingQueue.take(ArrayBlockingQueue.java:403) at org.apache.hadoop.hbase.io.hfile.bucket.BucketCache.getRAMQueueEntries(BucketCache.java:1168) at org.apache.hadoop.hbase.io.hfile.bucket.BucketCache$WriterThread.run(BucketCache.java:984) "main-BucketCacheWriter-1" #64 daemon prio=5 os_prio=0 tid=0x00007fc11c496000 nid=0x73b4 waiting on condition [0x00007fc11a9cc000] java.lang.Thread.State: WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007fc1bc1176d8> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) at java.util.concurrent.ArrayBlockingQueue.take(ArrayBlockingQueue.java:403) at org.apache.hadoop.hbase.io.hfile.bucket.BucketCache.getRAMQueueEntries(BucketCache.java:1168) at org.apache.hadoop.hbase.io.hfile.bucket.BucketCache$WriterThread.run(BucketCache.java:984) "main-BucketCacheWriter-0" #63 daemon prio=5 os_prio=0 tid=0x00007fc11c513000 nid=0x73b3 waiting on condition [0x00007fc11afec000] java.lang.Thread.State: WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007fc1bc0960b8> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) at java.util.concurrent.ArrayBlockingQueue.take(ArrayBlockingQueue.java:403) at org.apache.hadoop.hbase.io.hfile.bucket.BucketCache.getRAMQueueEntries(BucketCache.java:1168) at org.apache.hadoop.hbase.io.hfile.bucket.BucketCache$WriterThread.run(BucketCache.java:984) "LruBlockCacheStatsExecutor" #49 daemon prio=5 os_prio=0 tid=0x00007fc11d1ef800 nid=0x736a waiting on condition [0x00007fc11a4ae000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007fc1bc1178e0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1093) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:750) "main.LruBlockCache.EvictionThread" #47 daemon prio=5 os_prio=0 tid=0x00007fc11d1f3000 nid=0x7369 in Object.wait() [0x00007fc11b6ed000] java.lang.Thread.State: TIMED_WAITING (on object monitor) at java.lang.Object.wait(Native Method) at org.apache.hadoop.hbase.io.hfile.LruBlockCache$EvictionThread.run(LruBlockCache.java:942) - locked <0x00007fc1bc0ad8b8> (a org.apache.hadoop.hbase.io.hfile.LruBlockCache$EvictionThread) "RS-EventLoopGroup-1-1" #45 daemon prio=10 os_prio=0 tid=0x00007fc11cc3d800 nid=0x7367 runnable [0x00007fc1204fb000] java.lang.Thread.State: RUNNABLE at org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) at org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:157) at org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:150) at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:290) at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:347) at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:989) at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) at java.lang.Thread.run(Thread.java:750) "HBase-Metrics2-1" #41 daemon prio=5 os_prio=0 tid=0x00007fc11c8a4000 nid=0x7361 waiting on condition [0x00007fc11b9ee000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007fc1bc0962c0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1093) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:750) "SeedGenerator Thread" #14 daemon prio=1 os_prio=0 tid=0x00007fc11c51f000 nid=0x733f in Object.wait() [0x00007fc12060d000] java.lang.Thread.State: WAITING (on object monitor) at java.lang.Object.wait(Native Method) - waiting on <0x00007fc1bc0ada38> (a sun.security.provider.SeedGenerator$ThreadedSeedGenerator) at java.lang.Object.wait(Object.java:502) at sun.security.provider.SeedGenerator$ThreadedSeedGenerator.run(SeedGenerator.java:334) - locked <0x00007fc1bc0ada38> (a sun.security.provider.SeedGenerator$ThreadedSeedGenerator) at java.lang.Thread.run(Thread.java:750) "event filterer" #11 daemon prio=5 os_prio=0 tid=0x00007fc11c2e2800 nid=0x7339 waiting on condition [0x00007fc120d47000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007fc1bc117f50> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) at com.cloudera.cmf.eventcatcher.client.logs.LogEventProcessor.runFiltering(LogEventProcessor.java:132) at com.cloudera.cmf.eventcatcher.client.logs.LogEventProcessor.access$000(LogEventProcessor.java:28) at com.cloudera.cmf.eventcatcher.client.logs.LogEventProcessor$1.run(LogEventProcessor.java:81) "Service Thread" #10 daemon prio=9 os_prio=0 tid=0x00007fc99aaf2000 nid=0x7335 runnable [0x0000000000000000] java.lang.Thread.State: RUNNABLE "C1 CompilerThread3" #9 daemon prio=9 os_prio=0 tid=0x00007fc99aae4800 nid=0x7334 waiting on condition [0x0000000000000000] java.lang.Thread.State: RUNNABLE "C2 CompilerThread2" #8 daemon prio=9 os_prio=0 tid=0x00007fc99aada800 nid=0x7333 waiting on condition [0x0000000000000000] java.lang.Thread.State: RUNNABLE "C2 CompilerThread1" #7 daemon prio=9 os_prio=0 tid=0x00007fc99aad8000 nid=0x7332 waiting on condition [0x0000000000000000] java.lang.Thread.State: RUNNABLE "C2 CompilerThread0" #6 daemon prio=9 os_prio=0 tid=0x00007fc99aad6000 nid=0x7331 waiting on condition [0x0000000000000000] java.lang.Thread.State: RUNNABLE "Signal Dispatcher" #5 daemon prio=9 os_prio=0 tid=0x00007fc99aac8000 nid=0x7330 runnable [0x0000000000000000] java.lang.Thread.State: RUNNABLE "Surrogate Locker Thread (Concurrent GC)" #4 daemon prio=9 os_prio=0 tid=0x00007fc99aac6000 nid=0x732f waiting on condition [0x0000000000000000] java.lang.Thread.State: RUNNABLE "Finalizer" #3 daemon prio=8 os_prio=0 tid=0x00007fc99aa99000 nid=0x732e in Object.wait() [0x00007fc12af1d000] java.lang.Thread.State: WAITING (on object monitor) at java.lang.Object.wait(Native Method) at java.lang.ref.ReferenceQueue.remove(ReferenceQueue.java:144) - locked <0x00007fc1bc0ad8a8> (a java.lang.ref.ReferenceQueue$Lock) at java.lang.ref.ReferenceQueue.remove(ReferenceQueue.java:165) at java.lang.ref.Finalizer$FinalizerThread.run(Finalizer.java:188) "Reference Handler" #2 daemon prio=10 os_prio=0 tid=0x00007fc99aa94000 nid=0x732d in Object.wait() [0x00007fc12b01e000] java.lang.Thread.State: WAITING (on object monitor) at java.lang.Object.wait(Native Method) at java.lang.Object.wait(Object.java:502) at java.lang.ref.Reference.tryHandlePending(Reference.java:191) - locked <0x00007fc1bc0ad898> (a java.lang.ref.Reference$Lock) at java.lang.ref.Reference$ReferenceHandler.run(Reference.java:153) "main" #1 prio=5 os_prio=0 tid=0x00007fc99806f000 nid=0x72d9 in Object.wait() [0x00007fc9a0a6f000] java.lang.Thread.State: WAITING (on object monitor) at java.lang.Object.wait(Native Method) - waiting on <0x00007fc1bc118010> (a org.apache.hadoop.hbase.regionserver.HRegionServer) at java.lang.Thread.join(Thread.java:1257) - locked <0x00007fc1bc118010> (a org.apache.hadoop.hbase.regionserver.HRegionServer) at java.lang.Thread.join(Thread.java:1331) at org.apache.hadoop.hbase.regionserver.HRegionServerCommandLine.start(HRegionServerCommandLine.java:63) at org.apache.hadoop.hbase.regionserver.HRegionServerCommandLine.run(HRegionServerCommandLine.java:85) at org.apache.hadoop.util.ToolRunner.run(ToolRunner.java:81) at org.apache.hadoop.hbase.util.ServerCommandLine.doMain(ServerCommandLine.java:152) at org.apache.hadoop.hbase.regionserver.HRegionServer.main(HRegionServer.java:3223) "VM Thread" os_prio=0 tid=0x00007fc99aa8a800 nid=0x732c runnable "Gang worker#0 (Parallel GC Threads)" os_prio=0 tid=0x00007fc99807f800 nid=0x72da runnable "Gang worker#1 (Parallel GC Threads)" os_prio=0 tid=0x00007fc998081800 nid=0x72db runnable "Gang worker#2 (Parallel GC Threads)" os_prio=0 tid=0x00007fc998083800 nid=0x72dc runnable "Gang worker#3 (Parallel GC Threads)" os_prio=0 tid=0x00007fc998085800 nid=0x72dd runnable "Gang worker#4 (Parallel GC Threads)" os_prio=0 tid=0x00007fc998087000 nid=0x72de runnable "Gang worker#5 (Parallel GC Threads)" os_prio=0 tid=0x00007fc998089000 nid=0x72df runnable "Gang worker#6 (Parallel GC Threads)" os_prio=0 tid=0x00007fc99808b000 nid=0x72e0 runnable "Gang worker#7 (Parallel GC Threads)" os_prio=0 tid=0x00007fc99808d000 nid=0x72e2 runnable "G1 Main Concurrent Mark GC Thread" os_prio=0 tid=0x00007fc9980b8000 nid=0x72ec runnable "Gang worker#0 (G1 Parallel Marking Threads)" os_prio=0 tid=0x00007fc9980ba000 nid=0x72ed runnable "Gang worker#1 (G1 Parallel Marking Threads)" os_prio=0 tid=0x00007fc9980bc000 nid=0x72ee runnable "G1 Concurrent Refinement Thread#0" os_prio=0 tid=0x00007fc99809f000 nid=0x72eb runnable "G1 Concurrent Refinement Thread#1" os_prio=0 tid=0x00007fc99809d800 nid=0x72ea runnable "G1 Concurrent Refinement Thread#2" os_prio=0 tid=0x00007fc99809b800 nid=0x72e9 runnable "G1 Concurrent Refinement Thread#3" os_prio=0 tid=0x00007fc998099800 nid=0x72e8 runnable "G1 Concurrent Refinement Thread#4" os_prio=0 tid=0x00007fc998097800 nid=0x72e7 runnable "G1 Concurrent Refinement Thread#5" os_prio=0 tid=0x00007fc998095800 nid=0x72e6 runnable "G1 Concurrent Refinement Thread#6" os_prio=0 tid=0x00007fc998093800 nid=0x72e5 runnable "G1 Concurrent Refinement Thread#7" os_prio=0 tid=0x00007fc998091800 nid=0x72e4 runnable "G1 Concurrent Refinement Thread#8" os_prio=0 tid=0x00007fc998090000 nid=0x72e3 runnable "VM Periodic Task Thread" os_prio=0 tid=0x00007fc99aaf4000 nid=0x7336 waiting on condition JNI global references: 71870 __END_STACKS_SAMPLE__