E0407 14:36:52.453650 3648 JvmPauseMonitor.java:222] All threads: E0407 14:36:52.957446 3648 JvmPauseMonitor.java:224] "Thread-311429" Id=2047134 RUNNABLE (in native) E0407 14:36:52.957900 3648 JvmPauseMonitor.java:224] "Thread-311428" Id=2047133 TIMED_WAITING on [Lorg.apache.hadoop.hbase.client.ResultBoundedCompletionService$QueueingFuture;@61887a39 at java.lang.Object.wait(Native Method) - waiting on [Lorg.apache.hadoop.hbase.client.ResultBoundedCompletionService$QueueingFuture;@61887a39 at java.lang.Object.wait(Object.java:460) at java.util.concurrent.TimeUnit.timedWait(TimeUnit.java:348) at org.apache.hadoop.hbase.client.ResultBoundedCompletionService.pollForSpecificCompletedTask(ResultBoundedCompletionService.java:258) at org.apache.hadoop.hbase.client.ResultBoundedCompletionService.pollForFirstSuccessfullyCompletedTask(ResultBoundedCompletionService.java:214) at org.apache.hadoop.hbase.client.ScannerCallableWithReplicas.call(ScannerCallableWithReplicas.java:233) at org.apache.hadoop.hbase.client.ScannerCallableWithReplicas.call(ScannerCallableWithReplicas.java:62) at org.apache.hadoop.hbase.client.RpcRetryingCaller.callWithoutRetries(RpcRetryingCaller.java:210) ... E0407 14:36:52.958004 3648 JvmPauseMonitor.java:224] "Thread-311426" Id=2047131 RUNNABLE at org.apache.hadoop.hbase.ipc.RpcClientImpl$Connection.writeRequest(RpcClientImpl.java:914) at org.apache.hadoop.hbase.ipc.RpcClientImpl$Connection.tracedWriteRequest(RpcClientImpl.java:874) at org.apache.hadoop.hbase.ipc.RpcClientImpl.call(RpcClientImpl.java:1246) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callBlockingMethod(AbstractRpcClient.java:227) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$BlockingRpcChannelImplementation.callBlockingMethod(AbstractRpcClient.java:336) at org.apache.hadoop.hbase.protobuf.generated.ClientProtos$ClientService$BlockingStub.scan(ClientProtos.java:34094) at org.apache.hadoop.hbase.client.ScannerCallable.close(ScannerCallable.java:383) at org.apache.hadoop.hbase.client.ScannerCallable.call(ScannerCallable.java:200) ... E0407 14:36:52.958109 3648 JvmPauseMonitor.java:224] "Thread-311425" Id=2047130 RUNNABLE at org.apache.hadoop.hbase.ipc.RpcClientImpl$Connection.writeRequest(RpcClientImpl.java:914) at org.apache.hadoop.hbase.ipc.RpcClientImpl$Connection.tracedWriteRequest(RpcClientImpl.java:874) at org.apache.hadoop.hbase.ipc.RpcClientImpl.call(RpcClientImpl.java:1246) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callBlockingMethod(AbstractRpcClient.java:227) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$BlockingRpcChannelImplementation.callBlockingMethod(AbstractRpcClient.java:336) at org.apache.hadoop.hbase.protobuf.generated.ClientProtos$ClientService$BlockingStub.scan(ClientProtos.java:34094) at org.apache.hadoop.hbase.client.ScannerCallable.close(ScannerCallable.java:383) at org.apache.hadoop.hbase.client.ScannerCallable.call(ScannerCallable.java:200) ... E0407 14:36:52.958204 3648 JvmPauseMonitor.java:224] "hconnection-0x6379200-shared--pool1-t1726429" Id=2047126 RUNNABLE at org.apache.hadoop.hbase.ipc.RpcClientImpl$Connection.writeRequest(RpcClientImpl.java:914) at org.apache.hadoop.hbase.ipc.RpcClientImpl$Connection.tracedWriteRequest(RpcClientImpl.java:874) at org.apache.hadoop.hbase.ipc.RpcClientImpl.call(RpcClientImpl.java:1246) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callBlockingMethod(AbstractRpcClient.java:227) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$BlockingRpcChannelImplementation.callBlockingMethod(AbstractRpcClient.java:336) at org.apache.hadoop.hbase.protobuf.generated.ClientProtos$ClientService$BlockingStub.scan(ClientProtos.java:34094) at org.apache.hadoop.hbase.client.ScannerCallable.call(ScannerCallable.java:220) at org.apache.hadoop.hbase.client.ScannerCallable.call(ScannerCallable.java:65) ... Number of locked synchronizers = 1 - java.util.concurrent.ThreadPoolExecutor$Worker@3a0ad1f E0407 14:36:52.958303 3648 JvmPauseMonitor.java:224] "Thread-311424" Id=2047107 RUNNABLE at org.apache.hadoop.hbase.ipc.RpcClientImpl$Connection.writeRequest(RpcClientImpl.java:914) at org.apache.hadoop.hbase.ipc.RpcClientImpl$Connection.tracedWriteRequest(RpcClientImpl.java:874) at org.apache.hadoop.hbase.ipc.RpcClientImpl.call(RpcClientImpl.java:1246) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callBlockingMethod(AbstractRpcClient.java:227) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$BlockingRpcChannelImplementation.callBlockingMethod(AbstractRpcClient.java:336) at org.apache.hadoop.hbase.protobuf.generated.ClientProtos$ClientService$BlockingStub.scan(ClientProtos.java:34094) at org.apache.hadoop.hbase.client.ScannerCallable.close(ScannerCallable.java:383) at org.apache.hadoop.hbase.client.ScannerCallable.call(ScannerCallable.java:200) ... E0407 14:36:52.958392 3648 JvmPauseMonitor.java:224] "Thread-311423" Id=2046929 RUNNABLE at org.apache.hadoop.hbase.ipc.RpcClientImpl$Connection.writeRequest(RpcClientImpl.java:914) at org.apache.hadoop.hbase.ipc.RpcClientImpl$Connection.tracedWriteRequest(RpcClientImpl.java:874) at org.apache.hadoop.hbase.ipc.RpcClientImpl.call(RpcClientImpl.java:1246) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callBlockingMethod(AbstractRpcClient.java:227) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$BlockingRpcChannelImplementation.callBlockingMethod(AbstractRpcClient.java:336) at org.apache.hadoop.hbase.protobuf.generated.ClientProtos$ClientService$BlockingStub.scan(ClientProtos.java:34094) at org.apache.hadoop.hbase.client.ScannerCallable.close(ScannerCallable.java:383) at org.apache.hadoop.hbase.client.ScannerCallable.call(ScannerCallable.java:200) ... E0407 14:36:52.958473 3648 JvmPauseMonitor.java:224] "Thread-311422" Id=2046922 RUNNABLE at org.apache.hadoop.hbase.ipc.RpcClientImpl$Connection.writeRequest(RpcClientImpl.java:914) at org.apache.hadoop.hbase.ipc.RpcClientImpl$Connection.tracedWriteRequest(RpcClientImpl.java:874) at org.apache.hadoop.hbase.ipc.RpcClientImpl.call(RpcClientImpl.java:1246) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callBlockingMethod(AbstractRpcClient.java:227) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$BlockingRpcChannelImplementation.callBlockingMethod(AbstractRpcClient.java:336) at org.apache.hadoop.hbase.protobuf.generated.ClientProtos$ClientService$BlockingStub.scan(ClientProtos.java:34094) at org.apache.hadoop.hbase.client.ScannerCallable.close(ScannerCallable.java:383) at org.apache.hadoop.hbase.client.ScannerCallable.call(ScannerCallable.java:200) ... E0407 14:36:52.958551 3648 JvmPauseMonitor.java:224] "hconnection-0x6379200-shared--pool1-t1726190" Id=2046884 RUNNABLE at org.apache.hadoop.hbase.ipc.RpcClientImpl$Connection.writeRequest(RpcClientImpl.java:914) at org.apache.hadoop.hbase.ipc.RpcClientImpl$Connection.tracedWriteRequest(RpcClientImpl.java:874) at org.apache.hadoop.hbase.ipc.RpcClientImpl.call(RpcClientImpl.java:1246) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callBlockingMethod(AbstractRpcClient.java:227) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$BlockingRpcChannelImplementation.callBlockingMethod(AbstractRpcClient.java:336) at org.apache.hadoop.hbase.protobuf.generated.ClientProtos$ClientService$BlockingStub.scan(ClientProtos.java:34094) at org.apache.hadoop.hbase.client.ScannerCallable.openScanner(ScannerCallable.java:400) at org.apache.hadoop.hbase.client.ScannerCallable.call(ScannerCallable.java:204) ... Number of locked synchronizers = 1 - java.util.concurrent.ThreadPoolExecutor$Worker@6efb536d E0407 14:36:52.958631 3648 JvmPauseMonitor.java:224] "hconnection-0x6379200-shared--pool1-t1726187" Id=2046881 RUNNABLE at org.apache.hadoop.hbase.ipc.RpcClientImpl$Connection.writeRequest(RpcClientImpl.java:914) at org.apache.hadoop.hbase.ipc.RpcClientImpl$Connection.tracedWriteRequest(RpcClientImpl.java:874) at org.apache.hadoop.hbase.ipc.RpcClientImpl.call(RpcClientImpl.java:1246) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callBlockingMethod(AbstractRpcClient.java:227) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$BlockingRpcChannelImplementation.callBlockingMethod(AbstractRpcClient.java:336) at org.apache.hadoop.hbase.protobuf.generated.ClientProtos$ClientService$BlockingStub.scan(ClientProtos.java:34094) at org.apache.hadoop.hbase.client.ScannerCallable.call(ScannerCallable.java:220) at org.apache.hadoop.hbase.client.ScannerCallable.call(ScannerCallable.java:65) ... Number of locked synchronizers = 1 - java.util.concurrent.ThreadPoolExecutor$Worker@6aca9ba9 E0407 14:36:52.958724 3648 JvmPauseMonitor.java:224] "hconnection-0x6379200-shared--pool1-t1726186" Id=2046880 RUNNABLE at org.apache.hadoop.hbase.ipc.RpcClientImpl$Connection.writeRequest(RpcClientImpl.java:914) at org.apache.hadoop.hbase.ipc.RpcClientImpl$Connection.tracedWriteRequest(RpcClientImpl.java:874) at org.apache.hadoop.hbase.ipc.RpcClientImpl.call(RpcClientImpl.java:1246) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callBlockingMethod(AbstractRpcClient.java:227) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$BlockingRpcChannelImplementation.callBlockingMethod(AbstractRpcClient.java:336) at org.apache.hadoop.hbase.protobuf.generated.ClientProtos$ClientService$BlockingStub.scan(ClientProtos.java:34094) at org.apache.hadoop.hbase.client.ScannerCallable.call(ScannerCallable.java:220) at org.apache.hadoop.hbase.client.ScannerCallable.call(ScannerCallable.java:65) ... Number of locked synchronizers = 1 - java.util.concurrent.ThreadPoolExecutor$Worker@1d0c6009 E0407 14:36:52.958807 3648 JvmPauseMonitor.java:224] "hconnection-0x6379200-shared--pool1-t1726185" Id=2046879 RUNNABLE at org.apache.hadoop.hbase.ipc.RpcClientImpl$Connection.writeRequest(RpcClientImpl.java:914) at org.apache.hadoop.hbase.ipc.RpcClientImpl$Connection.tracedWriteRequest(RpcClientImpl.java:874) at org.apache.hadoop.hbase.ipc.RpcClientImpl.call(RpcClientImpl.java:1246) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callBlockingMethod(AbstractRpcClient.java:227) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$BlockingRpcChannelImplementation.callBlockingMethod(AbstractRpcClient.java:336) at org.apache.hadoop.hbase.protobuf.generated.ClientProtos$ClientService$BlockingStub.scan(ClientProtos.java:34094) at org.apache.hadoop.hbase.client.ScannerCallable.call(ScannerCallable.java:220) at org.apache.hadoop.hbase.client.ScannerCallable.call(ScannerCallable.java:65) ... Number of locked synchronizers = 1 - java.util.concurrent.ThreadPoolExecutor$Worker@14c05ae6 E0407 14:36:52.958881 3648 JvmPauseMonitor.java:224] "hconnection-0x6379200-shared--pool1-t1726180" Id=2046874 RUNNABLE at org.apache.hadoop.hbase.ipc.RpcClientImpl$Connection.writeRequest(RpcClientImpl.java:914) at org.apache.hadoop.hbase.ipc.RpcClientImpl$Connection.tracedWriteRequest(RpcClientImpl.java:874) at org.apache.hadoop.hbase.ipc.RpcClientImpl.call(RpcClientImpl.java:1246) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callBlockingMethod(AbstractRpcClient.java:227) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$BlockingRpcChannelImplementation.callBlockingMethod(AbstractRpcClient.java:336) at org.apache.hadoop.hbase.protobuf.generated.ClientProtos$ClientService$BlockingStub.scan(ClientProtos.java:34094) at org.apache.hadoop.hbase.client.ScannerCallable.call(ScannerCallable.java:220) at org.apache.hadoop.hbase.client.ScannerCallable.call(ScannerCallable.java:65) ... Number of locked synchronizers = 1 - java.util.concurrent.ThreadPoolExecutor$Worker@1ed9e9bb E0407 14:36:52.958956 3648 JvmPauseMonitor.java:224] "hconnection-0x6379200-shared--pool1-t1726179" Id=2046873 RUNNABLE at org.apache.hadoop.hbase.ipc.RpcClientImpl$Connection.writeRequest(RpcClientImpl.java:914) at org.apache.hadoop.hbase.ipc.RpcClientImpl$Connection.tracedWriteRequest(RpcClientImpl.java:874) at org.apache.hadoop.hbase.ipc.RpcClientImpl.call(RpcClientImpl.java:1246) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callBlockingMethod(AbstractRpcClient.java:227) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$BlockingRpcChannelImplementation.callBlockingMethod(AbstractRpcClient.java:336) at org.apache.hadoop.hbase.protobuf.generated.ClientProtos$ClientService$BlockingStub.scan(ClientProtos.java:34094) at org.apache.hadoop.hbase.client.ScannerCallable.call(ScannerCallable.java:220) at org.apache.hadoop.hbase.client.ScannerCallable.call(ScannerCallable.java:65) ... Number of locked synchronizers = 1 - java.util.concurrent.ThreadPoolExecutor$Worker@1ae66e3f E0407 14:36:52.959040 3648 JvmPauseMonitor.java:224] "Thread-311421" Id=2046872 RUNNABLE at org.apache.hadoop.hbase.ipc.RpcClientImpl$Connection.writeRequest(RpcClientImpl.java:914) at org.apache.hadoop.hbase.ipc.RpcClientImpl$Connection.tracedWriteRequest(RpcClientImpl.java:874) at org.apache.hadoop.hbase.ipc.RpcClientImpl.call(RpcClientImpl.java:1246) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callBlockingMethod(AbstractRpcClient.java:227) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$BlockingRpcChannelImplementation.callBlockingMethod(AbstractRpcClient.java:336) at org.apache.hadoop.hbase.protobuf.generated.ClientProtos$ClientService$BlockingStub.scan(ClientProtos.java:34094) at org.apache.hadoop.hbase.client.ScannerCallable.close(ScannerCallable.java:383) at org.apache.hadoop.hbase.client.ScannerCallable.call(ScannerCallable.java:200) ... E0407 14:36:52.959122 3648 JvmPauseMonitor.java:224] "hconnection-0x6379200-shared--pool1-t1726177" Id=2046870 RUNNABLE at org.apache.hadoop.hbase.ipc.RpcClientImpl$Connection.writeRequest(RpcClientImpl.java:914) at org.apache.hadoop.hbase.ipc.RpcClientImpl$Connection.tracedWriteRequest(RpcClientImpl.java:874) at org.apache.hadoop.hbase.ipc.RpcClientImpl.call(RpcClientImpl.java:1246) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callBlockingMethod(AbstractRpcClient.java:227) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$BlockingRpcChannelImplementation.callBlockingMethod(AbstractRpcClient.java:336) at org.apache.hadoop.hbase.protobuf.generated.ClientProtos$ClientService$BlockingStub.scan(ClientProtos.java:34094) at org.apache.hadoop.hbase.client.ScannerCallable.call(ScannerCallable.java:220) at org.apache.hadoop.hbase.client.ScannerCallable.call(ScannerCallable.java:65) ... Number of locked synchronizers = 1 - java.util.concurrent.ThreadPoolExecutor$Worker@6412b550 E0407 14:36:52.959201 3648 JvmPauseMonitor.java:224] "Thread-311420" Id=2046869 RUNNABLE at org.apache.hadoop.hbase.ipc.RpcClientImpl$Connection.writeRequest(RpcClientImpl.java:914) at org.apache.hadoop.hbase.ipc.RpcClientImpl$Connection.tracedWriteRequest(RpcClientImpl.java:874) at org.apache.hadoop.hbase.ipc.RpcClientImpl.call(RpcClientImpl.java:1246) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callBlockingMethod(AbstractRpcClient.java:227) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$BlockingRpcChannelImplementation.callBlockingMethod(AbstractRpcClient.java:336) at org.apache.hadoop.hbase.protobuf.generated.ClientProtos$ClientService$BlockingStub.scan(ClientProtos.java:34094) at org.apache.hadoop.hbase.client.ScannerCallable.close(ScannerCallable.java:383) at org.apache.hadoop.hbase.client.ScannerCallable.call(ScannerCallable.java:200) ... E0407 14:36:52.959280 3648 JvmPauseMonitor.java:224] "IPC Parameter Sending Thread #9200" Id=2046565 TIMED_WAITING on java.util.concurrent.SynchronousQueue$TransferStack@2a68327e at sun.misc.Unsafe.park(Native Method) - waiting on java.util.concurrent.SynchronousQueue$TransferStack@2a68327e at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460) at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362) at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1066) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) ... E0407 14:36:52.959414 3648 JvmPauseMonitor.java:224] "IPC Client (363509958) connection to namenode/ip:8020 from impala/host@EXAMPLE.COM" Id=2046564 RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked sun.nio.ch.Util$3@63cde4cb - locked java.util.Collections$UnmodifiableSet@4f17c3c1 - locked sun.nio.ch.EPollSelectorImpl@6dec31a8 at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) at org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:157) at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) ... E0407 14:36:52.959496 3648 JvmPauseMonitor.java:224] "IPC Client (363509958) connection to host/ip:60020 from impala/host@EXAMPLE.COM" Id=2041702 TIMED_WAITING on org.apache.hadoop.hbase.ipc.RpcClientImpl$Connection@7b638158 at java.lang.Object.wait(Native Method) - waiting on org.apache.hadoop.hbase.ipc.RpcClientImpl$Connection@7b638158 at org.apache.hadoop.hbase.ipc.RpcClientImpl$Connection.waitForWork(RpcClientImpl.java:551) at org.apache.hadoop.hbase.ipc.RpcClientImpl$Connection.run(RpcClientImpl.java:566) E0407 14:36:52.959555 3648 JvmPauseMonitor.java:224] "Thread-289310" Id=1914233 RUNNABLE (in native) E0407 14:36:52.959615 3648 JvmPauseMonitor.java:224] "Thread-215997" Id=1456002 TIMED_WAITING on java.util.LinkedList@770e8eed at java.lang.Object.wait(Native Method) - waiting on java.util.LinkedList@770e8eed at org.apache.hadoop.hdfs.DFSOutputStream$DataStreamer.run(DFSOutputStream.java:755) E0407 14:36:52.959679 3648 JvmPauseMonitor.java:224] "LeaseRenewer:impala@nameservice1" Id=1455999 TIMED_WAITING at java.lang.Thread.sleep(Native Method) at org.apache.hadoop.hdfs.LeaseRenewer.run(LeaseRenewer.java:419) at org.apache.hadoop.hdfs.LeaseRenewer.access$600(LeaseRenewer.java:72) at org.apache.hadoop.hdfs.LeaseRenewer$1.run(LeaseRenewer.java:302) at java.lang.Thread.run(Thread.java:745) E0407 14:36:52.959743 3648 JvmPauseMonitor.java:224] "LeaseRenewer:impala@nameservice1" Id=1453649 TIMED_WAITING at java.lang.Thread.sleep(Native Method) at org.apache.hadoop.hdfs.LeaseRenewer.run(LeaseRenewer.java:419) at org.apache.hadoop.hdfs.LeaseRenewer.access$600(LeaseRenewer.java:72) at org.apache.hadoop.hdfs.LeaseRenewer$1.run(LeaseRenewer.java:302) at java.lang.Thread.run(Thread.java:745) E0407 14:36:52.959798 3648 JvmPauseMonitor.java:224] "LeaseRenewer:impala@nameservice1" Id=1380547 TIMED_WAITING at java.lang.Thread.sleep(Native Method) at org.apache.hadoop.hdfs.LeaseRenewer.run(LeaseRenewer.java:419) at org.apache.hadoop.hdfs.LeaseRenewer.access$600(LeaseRenewer.java:72) at org.apache.hadoop.hdfs.LeaseRenewer$1.run(LeaseRenewer.java:302) at java.lang.Thread.run(Thread.java:745) E0407 14:36:52.959858 3648 JvmPauseMonitor.java:224] "LeaseRenewer:impala@nameservice1" Id=1347056 TIMED_WAITING at java.lang.Thread.sleep(Native Method) at org.apache.hadoop.hdfs.LeaseRenewer.run(LeaseRenewer.java:419) at org.apache.hadoop.hdfs.LeaseRenewer.access$600(LeaseRenewer.java:72) at org.apache.hadoop.hdfs.LeaseRenewer$1.run(LeaseRenewer.java:302) at java.lang.Thread.run(Thread.java:745) E0407 14:36:52.959916 3648 JvmPauseMonitor.java:224] "LeaseRenewer:impala@nameservice1" Id=1307232 TIMED_WAITING at java.lang.Thread.sleep(Native Method) at org.apache.hadoop.hdfs.LeaseRenewer.run(LeaseRenewer.java:419) at org.apache.hadoop.hdfs.LeaseRenewer.access$600(LeaseRenewer.java:72) at org.apache.hadoop.hdfs.LeaseRenewer$1.run(LeaseRenewer.java:302) at java.lang.Thread.run(Thread.java:745) E0407 14:36:52.959990 3648 JvmPauseMonitor.java:224] "LeaseRenewer:impala@nameservice1" Id=1231537 TIMED_WAITING at java.lang.Thread.sleep(Native Method) at org.apache.hadoop.hdfs.LeaseRenewer.run(LeaseRenewer.java:419) at org.apache.hadoop.hdfs.LeaseRenewer.access$600(LeaseRenewer.java:72) at org.apache.hadoop.hdfs.LeaseRenewer$1.run(LeaseRenewer.java:302) at java.lang.Thread.run(Thread.java:745) E0407 14:36:52.967484 3648 JvmPauseMonitor.java:224] "LeaseRenewer:impala@nameservice1" Id=1205719 TIMED_WAITING at java.lang.Thread.sleep(Native Method) at org.apache.hadoop.hdfs.LeaseRenewer.run(LeaseRenewer.java:419) at org.apache.hadoop.hdfs.LeaseRenewer.access$600(LeaseRenewer.java:72) at org.apache.hadoop.hdfs.LeaseRenewer$1.run(LeaseRenewer.java:302) at java.lang.Thread.run(Thread.java:745) E0407 14:36:52.967650 3648 JvmPauseMonitor.java:224] "LeaseRenewer:impala@nameservice1" Id=1194992 TIMED_WAITING at java.lang.Thread.sleep(Native Method) at org.apache.hadoop.hdfs.LeaseRenewer.run(LeaseRenewer.java:419) at org.apache.hadoop.hdfs.LeaseRenewer.access$600(LeaseRenewer.java:72) at org.apache.hadoop.hdfs.LeaseRenewer$1.run(LeaseRenewer.java:302) at java.lang.Thread.run(Thread.java:745) E0407 14:36:52.967752 3648 JvmPauseMonitor.java:224] "LeaseRenewer:impala@nameservice1" Id=1153944 TIMED_WAITING at java.lang.Thread.sleep(Native Method) at org.apache.hadoop.hdfs.LeaseRenewer.run(LeaseRenewer.java:419) at org.apache.hadoop.hdfs.LeaseRenewer.access$600(LeaseRenewer.java:72) at org.apache.hadoop.hdfs.LeaseRenewer$1.run(LeaseRenewer.java:302) at java.lang.Thread.run(Thread.java:745) E0407 14:36:52.967844 3648 JvmPauseMonitor.java:224] "LeaseRenewer:impala@nameservice1" Id=1104284 TIMED_WAITING at java.lang.Thread.sleep(Native Method) at org.apache.hadoop.hdfs.LeaseRenewer.run(LeaseRenewer.java:419) at org.apache.hadoop.hdfs.LeaseRenewer.access$600(LeaseRenewer.java:72) at org.apache.hadoop.hdfs.LeaseRenewer$1.run(LeaseRenewer.java:302) at java.lang.Thread.run(Thread.java:745) E0407 14:36:52.967911 3648 JvmPauseMonitor.java:224] "LeaseRenewer:impala@nameservice1" Id=979320 TIMED_WAITING at java.lang.Thread.sleep(Native Method) at org.apache.hadoop.hdfs.LeaseRenewer.run(LeaseRenewer.java:419) at org.apache.hadoop.hdfs.LeaseRenewer.access$600(LeaseRenewer.java:72) at org.apache.hadoop.hdfs.LeaseRenewer$1.run(LeaseRenewer.java:302) at java.lang.Thread.run(Thread.java:745) E0407 14:36:52.967978 3648 JvmPauseMonitor.java:224] "LeaseRenewer:impala@nameservice1" Id=966934 TIMED_WAITING at java.lang.Thread.sleep(Native Method) at org.apache.hadoop.hdfs.LeaseRenewer.run(LeaseRenewer.java:419) at org.apache.hadoop.hdfs.LeaseRenewer.access$600(LeaseRenewer.java:72) at org.apache.hadoop.hdfs.LeaseRenewer$1.run(LeaseRenewer.java:302) at java.lang.Thread.run(Thread.java:745) E0407 14:36:52.968080 3648 JvmPauseMonitor.java:224] "LeaseRenewer:impala@nameservice1" Id=776748 TIMED_WAITING at java.lang.Thread.sleep(Native Method) at org.apache.hadoop.hdfs.LeaseRenewer.run(LeaseRenewer.java:419) at org.apache.hadoop.hdfs.LeaseRenewer.access$600(LeaseRenewer.java:72) at org.apache.hadoop.hdfs.LeaseRenewer$1.run(LeaseRenewer.java:302) at java.lang.Thread.run(Thread.java:745) E0407 14:36:52.968175 3648 JvmPauseMonitor.java:224] "LeaseRenewer:impala@nameservice1" Id=660141 TIMED_WAITING at java.lang.Thread.sleep(Native Method) at org.apache.hadoop.hdfs.LeaseRenewer.run(LeaseRenewer.java:419) at org.apache.hadoop.hdfs.LeaseRenewer.access$600(LeaseRenewer.java:72) at org.apache.hadoop.hdfs.LeaseRenewer$1.run(LeaseRenewer.java:302) at java.lang.Thread.run(Thread.java:745) E0407 14:36:52.968245 3648 JvmPauseMonitor.java:224] "LeaseRenewer:impala@nameservice1" Id=612276 TIMED_WAITING at java.lang.Thread.sleep(Native Method) at org.apache.hadoop.hdfs.LeaseRenewer.run(LeaseRenewer.java:419) at org.apache.hadoop.hdfs.LeaseRenewer.access$600(LeaseRenewer.java:72) at org.apache.hadoop.hdfs.LeaseRenewer$1.run(LeaseRenewer.java:302) at java.lang.Thread.run(Thread.java:745) E0407 14:36:52.968354 3648 JvmPauseMonitor.java:224] "LeaseRenewer:impala@nameservice1" Id=588471 TIMED_WAITING at java.lang.Thread.sleep(Native Method) at org.apache.hadoop.hdfs.LeaseRenewer.run(LeaseRenewer.java:419) at org.apache.hadoop.hdfs.LeaseRenewer.access$600(LeaseRenewer.java:72) at org.apache.hadoop.hdfs.LeaseRenewer$1.run(LeaseRenewer.java:302) at java.lang.Thread.run(Thread.java:745) E0407 14:36:52.968442 3648 JvmPauseMonitor.java:224] "LeaseRenewer:impala@nameservice1" Id=582434 TIMED_WAITING at java.lang.Thread.sleep(Native Method) at org.apache.hadoop.hdfs.LeaseRenewer.run(LeaseRenewer.java:419) at org.apache.hadoop.hdfs.LeaseRenewer.access$600(LeaseRenewer.java:72) at org.apache.hadoop.hdfs.LeaseRenewer$1.run(LeaseRenewer.java:302) at java.lang.Thread.run(Thread.java:745) E0407 14:36:52.968521 3648 JvmPauseMonitor.java:224] "LeaseRenewer:impala@nameservice1" Id=540369 TIMED_WAITING at java.lang.Thread.sleep(Native Method) at org.apache.hadoop.hdfs.LeaseRenewer.run(LeaseRenewer.java:419) at org.apache.hadoop.hdfs.LeaseRenewer.access$600(LeaseRenewer.java:72) at org.apache.hadoop.hdfs.LeaseRenewer$1.run(LeaseRenewer.java:302) at java.lang.Thread.run(Thread.java:745) E0407 14:36:52.968585 3648 JvmPauseMonitor.java:224] "LeaseRenewer:impala@nameservice1" Id=508389 TIMED_WAITING at java.lang.Thread.sleep(Native Method) at org.apache.hadoop.hdfs.LeaseRenewer.run(LeaseRenewer.java:419) at org.apache.hadoop.hdfs.LeaseRenewer.access$600(LeaseRenewer.java:72) at org.apache.hadoop.hdfs.LeaseRenewer$1.run(LeaseRenewer.java:302) at java.lang.Thread.run(Thread.java:745) E0407 14:36:52.968713 3648 JvmPauseMonitor.java:224] "LeaseRenewer:impala@nameservice1" Id=474461 TIMED_WAITING at java.lang.Thread.sleep(Native Method) at org.apache.hadoop.hdfs.LeaseRenewer.run(LeaseRenewer.java:419) at org.apache.hadoop.hdfs.LeaseRenewer.access$600(LeaseRenewer.java:72) at org.apache.hadoop.hdfs.LeaseRenewer$1.run(LeaseRenewer.java:302) at java.lang.Thread.run(Thread.java:745) E0407 14:36:52.968858 3648 JvmPauseMonitor.java:224] "LeaseRenewer:impala@nameservice1" Id=381177 TIMED_WAITING at java.lang.Thread.sleep(Native Method) at org.apache.hadoop.hdfs.LeaseRenewer.run(LeaseRenewer.java:419) at org.apache.hadoop.hdfs.LeaseRenewer.access$600(LeaseRenewer.java:72) at org.apache.hadoop.hdfs.LeaseRenewer$1.run(LeaseRenewer.java:302) at java.lang.Thread.run(Thread.java:745) E0407 14:36:52.969018 3648 JvmPauseMonitor.java:224] "LeaseRenewer:impala@nameservice1" Id=352355 TIMED_WAITING at java.lang.Thread.sleep(Native Method) at org.apache.hadoop.hdfs.LeaseRenewer.run(LeaseRenewer.java:419) at org.apache.hadoop.hdfs.LeaseRenewer.access$600(LeaseRenewer.java:72) at org.apache.hadoop.hdfs.LeaseRenewer$1.run(LeaseRenewer.java:302) at java.lang.Thread.run(Thread.java:745) E0407 14:36:52.969157 3648 JvmPauseMonitor.java:224] "LeaseRenewer:impala@nameservice1" Id=342014 TIMED_WAITING at java.lang.Thread.sleep(Native Method) at org.apache.hadoop.hdfs.LeaseRenewer.run(LeaseRenewer.java:419) at org.apache.hadoop.hdfs.LeaseRenewer.access$600(LeaseRenewer.java:72) at org.apache.hadoop.hdfs.LeaseRenewer$1.run(LeaseRenewer.java:302) at java.lang.Thread.run(Thread.java:745) E0407 14:36:52.969287 3648 JvmPauseMonitor.java:224] "LeaseRenewer:impala@nameservice1" Id=265269 TIMED_WAITING at java.lang.Thread.sleep(Native Method) at org.apache.hadoop.hdfs.LeaseRenewer.run(LeaseRenewer.java:419) at org.apache.hadoop.hdfs.LeaseRenewer.access$600(LeaseRenewer.java:72) at org.apache.hadoop.hdfs.LeaseRenewer$1.run(LeaseRenewer.java:302) at java.lang.Thread.run(Thread.java:745) E0407 14:36:52.969398 3648 JvmPauseMonitor.java:224] "LeaseRenewer:impala@nameservice1" Id=140768 TIMED_WAITING at java.lang.Thread.sleep(Native Method) at org.apache.hadoop.hdfs.LeaseRenewer.run(LeaseRenewer.java:419) at org.apache.hadoop.hdfs.LeaseRenewer.access$600(LeaseRenewer.java:72) at org.apache.hadoop.hdfs.LeaseRenewer$1.run(LeaseRenewer.java:302) at java.lang.Thread.run(Thread.java:745) E0407 14:36:52.969485 3648 JvmPauseMonitor.java:224] "LeaseRenewer:impala@nameservice1" Id=86896 TIMED_WAITING at java.lang.Thread.sleep(Native Method) at org.apache.hadoop.hdfs.LeaseRenewer.run(LeaseRenewer.java:419) at org.apache.hadoop.hdfs.LeaseRenewer.access$600(LeaseRenewer.java:72) at org.apache.hadoop.hdfs.LeaseRenewer$1.run(LeaseRenewer.java:302) at java.lang.Thread.run(Thread.java:745) E0407 14:36:52.969584 3648 JvmPauseMonitor.java:224] "LeaseRenewer:impala@nameservice1" Id=76053 TIMED_WAITING at java.lang.Thread.sleep(Native Method) at org.apache.hadoop.hdfs.LeaseRenewer.run(LeaseRenewer.java:419) at org.apache.hadoop.hdfs.LeaseRenewer.access$600(LeaseRenewer.java:72) at org.apache.hadoop.hdfs.LeaseRenewer$1.run(LeaseRenewer.java:302) at java.lang.Thread.run(Thread.java:745) E0407 14:36:52.969676 3648 JvmPauseMonitor.java:224] "LeaseRenewer:impala@nameservice1" Id=27358 TIMED_WAITING at java.lang.Thread.sleep(Native Method) at org.apache.hadoop.hdfs.LeaseRenewer.run(LeaseRenewer.java:419) at org.apache.hadoop.hdfs.LeaseRenewer.access$600(LeaseRenewer.java:72) at org.apache.hadoop.hdfs.LeaseRenewer$1.run(LeaseRenewer.java:302) at java.lang.Thread.run(Thread.java:745) E0407 14:36:52.969760 3648 JvmPauseMonitor.java:224] "Thread-99-EventThread" Id=135 WAITING on java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject@11b98ce8 at sun.misc.Unsafe.park(Native Method) - waiting on java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject@11b98ce8 at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) at org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:506) E0407 14:36:52.969868 3648 JvmPauseMonitor.java:224] "Thread-99-SendThread(baelish-002.edh.cloudera.com:2181)" Id=134 RUNNABLE (in native) at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked sun.nio.ch.Util$3@4162df14 - locked java.util.Collections$UnmodifiableSet@26ecff21 - locked sun.nio.ch.EPollSelectorImpl@251674f7 at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.zookeeper.ClientCnxnSocketNIO.doTransport(ClientCnxnSocketNIO.java:338) at org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1126) E0407 14:36:52.969954 3648 JvmPauseMonitor.java:224] "ShortCircuitCache_SlotReleaser" Id=96 WAITING on java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject@78156271 at sun.misc.Unsafe.park(Native Method) - waiting on java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject@78156271 at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1081) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1067) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) ... E0407 14:36:52.970005 3648 JvmPauseMonitor.java:224] "Thread-44" Id=71 RUNNABLE (in native) E0407 14:36:52.970060 3648 JvmPauseMonitor.java:224] "Thread-41" Id=68 RUNNABLE (in native) E0407 14:36:52.970108 3648 JvmPauseMonitor.java:224] "Thread-40" Id=67 RUNNABLE E0407 14:36:52.970172 3648 JvmPauseMonitor.java:224] "Thread-39" Id=66 RUNNABLE (in native) E0407 14:36:52.970275 3648 JvmPauseMonitor.java:224] "Thread-38" Id=65 RUNNABLE (in native) E0407 14:36:52.970358 3648 JvmPauseMonitor.java:224] "Thread-36" Id=63 RUNNABLE (in native) E0407 14:36:52.970412 3648 JvmPauseMonitor.java:224] "Thread-35" Id=62 RUNNABLE (in native) E0407 14:36:52.970456 3648 JvmPauseMonitor.java:224] "Thread-34" Id=61 RUNNABLE (in native) E0407 14:36:52.970553 3648 JvmPauseMonitor.java:224] "Thread-33" Id=60 RUNNABLE (in native) E0407 14:36:52.970620 3648 JvmPauseMonitor.java:224] "Thread-32" Id=59 RUNNABLE (in native) E0407 14:36:52.970705 3648 JvmPauseMonitor.java:224] "Thread-31" Id=58 RUNNABLE at sun.nio.ch.FileDispatcherImpl.pread0(Native Method) at sun.nio.ch.FileDispatcherImpl.pread(FileDispatcherImpl.java:52) at sun.nio.ch.IOUtil.readIntoNativeBuffer(IOUtil.java:220) at sun.nio.ch.IOUtil.read(IOUtil.java:197) at sun.nio.ch.FileChannelImpl.readInternal(FileChannelImpl.java:741) at sun.nio.ch.FileChannelImpl.read(FileChannelImpl.java:727) at org.apache.hadoop.hdfs.server.datanode.BlockMetadataHeader.preadHeader(BlockMetadataHeader.java:124) at org.apache.hadoop.hdfs.shortcircuit.ShortCircuitReplica.(ShortCircuitReplica.java:126) ... E0407 14:36:52.970819 3648 JvmPauseMonitor.java:224] "Thread-30" Id=57 WAITING on org.apache.hadoop.ipc.Client$Call@782b1fcf at java.lang.Object.wait(Native Method) - waiting on org.apache.hadoop.ipc.Client$Call@782b1fcf at java.lang.Object.wait(Object.java:502) at org.apache.hadoop.ipc.Client.call(Client.java:1495) at org.apache.hadoop.ipc.Client.call(Client.java:1441) at org.apache.hadoop.ipc.ProtobufRpcEngine$Invoker.invoke(ProtobufRpcEngine.java:231) at com.sun.proxy.$Proxy10.getBlockLocations(Unknown Source) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.getBlockLocations(ClientNamenodeProtocolTranslatorPB.java:268) at sun.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ... E0407 14:36:52.970881 3648 JvmPauseMonitor.java:224] "Thread-29" Id=56 RUNNABLE E0407 14:36:52.970957 3648 JvmPauseMonitor.java:224] "Thread-28" Id=55 WAITING on org.apache.hadoop.ipc.Client$Call@400db97e at java.lang.Object.wait(Native Method) - waiting on org.apache.hadoop.ipc.Client$Call@400db97e at java.lang.Object.wait(Object.java:502) at org.apache.hadoop.ipc.Client.call(Client.java:1495) at org.apache.hadoop.ipc.Client.call(Client.java:1441) at org.apache.hadoop.ipc.ProtobufRpcEngine$Invoker.invoke(ProtobufRpcEngine.java:231) at com.sun.proxy.$Proxy10.getBlockLocations(Unknown Source) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.getBlockLocations(ClientNamenodeProtocolTranslatorPB.java:268) at sun.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ... E0407 14:36:52.971004 3648 JvmPauseMonitor.java:224] "Thread-27" Id=54 RUNNABLE (in native) E0407 14:36:52.971086 3648 JvmPauseMonitor.java:224] "Thread-26" Id=53 WAITING on org.apache.hadoop.ipc.Client$Call@6d00b88b at java.lang.Object.wait(Native Method) - waiting on org.apache.hadoop.ipc.Client$Call@6d00b88b at java.lang.Object.wait(Object.java:502) at org.apache.hadoop.ipc.Client.call(Client.java:1495) at org.apache.hadoop.ipc.Client.call(Client.java:1441) at org.apache.hadoop.ipc.ProtobufRpcEngine$Invoker.invoke(ProtobufRpcEngine.java:231) at com.sun.proxy.$Proxy10.getBlockLocations(Unknown Source) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.getBlockLocations(ClientNamenodeProtocolTranslatorPB.java:268) at sun.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ... E0407 14:36:52.971259 3648 JvmPauseMonitor.java:224] "Thread-25" Id=52 RUNNABLE E0407 14:36:52.971374 3648 JvmPauseMonitor.java:224] "Thread-24" Id=51 WAITING on org.apache.hadoop.ipc.Client$Call@1c2b4401 at java.lang.Object.wait(Native Method) - waiting on org.apache.hadoop.ipc.Client$Call@1c2b4401 at java.lang.Object.wait(Object.java:502) at org.apache.hadoop.ipc.Client.call(Client.java:1495) at org.apache.hadoop.ipc.Client.call(Client.java:1441) at org.apache.hadoop.ipc.ProtobufRpcEngine$Invoker.invoke(ProtobufRpcEngine.java:231) at com.sun.proxy.$Proxy10.getBlockLocations(Unknown Source) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.getBlockLocations(ClientNamenodeProtocolTranslatorPB.java:268) at sun.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ... E0407 14:36:52.971447 3648 JvmPauseMonitor.java:224] "Thread-23" Id=50 RUNNABLE (in native) E0407 14:36:52.971536 3648 JvmPauseMonitor.java:224] "ShortCircuitCache_Cleaner" Id=49 TIMED_WAITING on java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject@79a7c572 at sun.misc.Unsafe.park(Native Method) - waiting on java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject@79a7c572 at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1093) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1067) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) ... E0407 14:36:52.971634 3648 JvmPauseMonitor.java:224] "org.apache.hadoop.hdfs.PeerCache@ad3cf0c" Id=48 TIMED_WAITING at java.lang.Thread.sleep(Native Method) at org.apache.hadoop.hdfs.PeerCache.run(PeerCache.java:255) at org.apache.hadoop.hdfs.PeerCache.access$000(PeerCache.java:46) at org.apache.hadoop.hdfs.PeerCache$1.run(PeerCache.java:124) at java.lang.Thread.run(Thread.java:745) E0407 14:36:52.971695 3648 JvmPauseMonitor.java:224] "Thread-20" Id=45 RUNNABLE (in native) E0407 14:36:52.971742 3648 JvmPauseMonitor.java:224] "Thread-19" Id=44 RUNNABLE (in native) E0407 14:36:52.971788 3648 JvmPauseMonitor.java:224] "Thread-18" Id=43 RUNNABLE (in native) E0407 14:36:52.971896 3648 JvmPauseMonitor.java:224] "Thread-17" Id=42 RUNNABLE (in native) E0407 14:36:52.971963 3648 JvmPauseMonitor.java:224] "Thread-16" Id=41 RUNNABLE (in native) E0407 14:36:52.972009 3648 JvmPauseMonitor.java:224] "Thread-15" Id=40 RUNNABLE (in native) E0407 14:36:52.972069 3648 JvmPauseMonitor.java:224] "Thread-14" Id=39 RUNNABLE (in native) E0407 14:36:52.972131 3648 JvmPauseMonitor.java:224] "Thread-13" Id=38 RUNNABLE (in native) E0407 14:36:52.972219 3648 JvmPauseMonitor.java:224] "Thread-9" Id=34 RUNNABLE (in native) E0407 14:36:52.972326 3648 JvmPauseMonitor.java:224] "FileWatchThread(/var/run/cloudera-scm-agent/process/105519-impala-IMPALAD/impala-conf/llama-site.xml)-0" Id=30 TIMED_WAITING on java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject@110bfe2d at sun.misc.Unsafe.park(Native Method) - waiting on java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject@110bfe2d at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1093) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1067) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) ... E0407 14:36:52.972463 3648 JvmPauseMonitor.java:224] "AllocationFileReloader" Id=29 TIMED_WAITING at java.lang.Thread.sleep(Native Method) at org.apache.impala.yarn.server.resourcemanager.scheduler.fair.AllocationFileLoaderService$1.run(AllocationFileLoaderService.java:132) E0407 14:36:52.972564 3648 JvmPauseMonitor.java:224] "Thread-4" Id=27 RUNNABLE (in native) at org.apache.hadoop.net.unix.DomainSocketWatcher.doPoll0(Native Method) at org.apache.hadoop.net.unix.DomainSocketWatcher.access$900(DomainSocketWatcher.java:52) at org.apache.hadoop.net.unix.DomainSocketWatcher$2.run(DomainSocketWatcher.java:501) at java.lang.Thread.run(Thread.java:745) E0407 14:36:52.972638 3648 JvmPauseMonitor.java:224] "org.apache.hadoop.fs.FileSystem$Statistics$StatisticsDataReferenceCleaner" Id=26 WAITING on java.lang.ref.ReferenceQueue$Lock@2614f92f at java.lang.Object.wait(Native Method) - waiting on java.lang.ref.ReferenceQueue$Lock@2614f92f at java.lang.ref.ReferenceQueue.remove(ReferenceQueue.java:143) at java.lang.ref.ReferenceQueue.remove(ReferenceQueue.java:164) at org.apache.hadoop.fs.FileSystem$Statistics$StatisticsDataReferenceCleaner.run(FileSystem.java:3213) at java.lang.Thread.run(Thread.java:745) E0407 14:36:52.972728 3648 JvmPauseMonitor.java:224] "JVM pause monitor" Id=25 RUNNABLE at sun.management.ThreadImpl.dumpThreads0(Native Method) at sun.management.ThreadImpl.dumpAllThreads(ThreadImpl.java:454) at org.apache.impala.util.JvmPauseMonitor$Monitor.checkForDeadlocks(JvmPauseMonitor.java:223) at org.apache.impala.util.JvmPauseMonitor$Monitor.run(JvmPauseMonitor.java:194) at java.lang.Thread.run(Thread.java:745) E0407 14:36:52.972805 3648 JvmPauseMonitor.java:224] "TGT Renewer for impala/host@EXAMPLE.COM" Id=24 TIMED_WAITING at java.lang.Thread.sleep(Native Method) at org.apache.hadoop.security.UserGroupInformation$1.run(UserGroupInformation.java:1022) at java.lang.Thread.run(Thread.java:745) E0407 14:36:52.972873 3648 JvmPauseMonitor.java:224] "Signal Dispatcher" Id=4 RUNNABLE E0407 14:36:52.972993 3648 JvmPauseMonitor.java:224] "Finalizer" Id=3 WAITING on java.lang.ref.ReferenceQueue$Lock@60d9aaa0 at java.lang.Object.wait(Native Method) - waiting on java.lang.ref.ReferenceQueue$Lock@60d9aaa0 at java.lang.ref.ReferenceQueue.remove(ReferenceQueue.java:143) at java.lang.ref.ReferenceQueue.remove(ReferenceQueue.java:164) at java.lang.ref.Finalizer$FinalizerThread.run(Finalizer.java:209) E0407 14:36:52.973083 3648 JvmPauseMonitor.java:224] "Reference Handler" Id=2 WAITING on java.lang.ref.Reference$Lock@27927f9c at java.lang.Object.wait(Native Method) - waiting on java.lang.ref.Reference$Lock@27927f9c at java.lang.Object.wait(Object.java:502) at java.lang.ref.Reference.tryHandlePending(Reference.java:191) at java.lang.ref.Reference$ReferenceHandler.run(Reference.java:153) E0407 14:36:52.973143 3648 JvmPauseMonitor.java:224] "main" Id=1 RUNNABLE (in native) F0407 14:36:52.973263 3648 JvmPauseMonitor.java:228] Aborting because of deadlocked threads in JVM. *** Check failure stack trace: *** @ 0x2149c4d @ 0x214b4f2 @ 0x2149627 @ 0x2149989 @ 0xd3abba @ 0x7fc170f8032e (RemoteException.java:73) at org.apache.hadoop.hdfs.DFSOutputStream$DataStreamer.locateFollowingBlock(DFSOutputStream.java:1878) at org.apache.hadoop.hdfs.DFSOutputStream$DataStreamer.nextBlockOutputStream(DFSOutputStream.java:1671) at org.apache.hadoop.hdfs.DFSOutputStream$DataStreamer.run(DFSOutputStream.java:790) Caused by: org.apache.hadoop.ipc.RemoteException(org.apache.hadoop.hdfs.protocol.DSQuotaExceededException): The DiskSpace quota of /user/user1 is exceeded: quota = 10737418240 B = 10 GB but diskspace consumed = 11383430874 B = 10.60 GB at org.apache.hadoop.hdfs.server.namenode.DirectoryWithQuotaFeature.verifyDiskspaceQuota(DirectoryWithQuotaFeature.java:149) at org.apache.hadoop.hdfs.server.namenode.DirectoryWithQuotaFeature.verifyQuota(DirectoryWithQuotaFeature.java:159) at org.apache.hadoop.hdfs.server.namenode.FSDirectory.verifyQuota(FSDirectory.java:2124) at org.apache.hadoop.hdfs.server.namenode.FSDirectory.updateCount(FSDirectory.java:1991) at org.apache.hadoop.hdfs.server.namenode.FSDirectory.updateCount(FSDirectory.java:1966) at org.apache.hadoop.hdfs.server.namenode.FSDirectory.addBlock(FSDirectory.java:463) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.saveAllocatedBlock(FSNamesystem.java:3962) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.getAdditionalBlock(FSNamesystem.java:3550) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.addBlock(NameNodeRpcServer.java:694) at org.apache.hadoop.hdfs.server.namenode.AuthorizationProviderProxyClientProtocol.addBlock(AuthorizationProviderProxyClientProtocol.java:219) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.addBlock(ClientNamenodeProtocolServerSideTranslatorPB.java:507) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine.java:617) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1073) at org.apache.hadoop.ipc.Server$Handler$1.run(Server.java:2278) at org.apache.hadoop.ipc.Server$Handler$1.run(Server.java:2274) at java.security.AccessController.doPrivileged(Native Method) at javax.security.auth.Subject.doAs(Subject.java:422) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1924) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:2272) at org.apache.hadoop.ipc.Client.call(Client.java:1504) at org.apache.hadoop.ipc.Client.call(Client.java:1441) at org.apache.hadoop.ipc.ProtobufRpcEngine$Invoker.invoke(ProtobufRpcEngine.java:231) at com.sun.proxy.$Proxy10.addBlock(Unknown Source) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.addBlock(ClientNamenodeProtocolTranslatorPB.java:425) at sun.reflect.GeneratedMethodAccessor10.invoke(Unknown Source) at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.lang.reflect.Method.invoke(Method.java:498) at org.apache.hadoop.io.retry.RetryInvocationHandler.invokeMethod(RetryInvocationHandler.java:258) at org.apache.hadoop.io.retry.RetryInvocationHandler.invoke(RetryInvocationHandler.java:104) at com.sun.proxy.$Proxy11.addBlock(Unknown Source) at org.apache.hadoop.hdfs.DFSOutputStream$DataStreamer.locateFollowingBlock(DFSOutputStream.java:1875) ... 2 more Wrote minidump to /var/log/impala-minidumps/impalad/06eaffcd-3c37-4616-808c50a0-46fffe1c.dmp