Log file created at: 2016/01/20 22:59:28 Running on machine: ux-reporting-engine-master-1-prod-us-east-1a Log line format: [IWEF]mmdd hh:mm:ss.uuuuuu threadid file:line] msg E0120 22:59:28.422256 75437 logging.cc:119] stderr will be logged to this file. E0121 01:56:44.168965 73220 data-stream-sender.cc:278] channel send status: Couldn't open transport for server-7-prod-us-east-1a:22222 (connect() failed: Connection refused) E0121 01:56:44.178045 71153 data-stream-sender.cc:278] channel send status: Couldn't open transport for server-7-prod-us-east-1a:22222 (connect() failed: Connection refused) E0121 01:56:44.716106 74774 data-stream-sender.cc:278] channel send status: Couldn't open transport for server-7-prod-us-east-1a:22222 (connect() failed: Connection refused) E0121 01:56:45.020022 74843 data-stream-sender.cc:278] channel send status: Couldn't open transport for server-7-prod-us-east-1a:22222 (connect() failed: Connection refused) E0121 01:56:45.178716 73220 data-stream-sender.cc:278] channel send status: Couldn't open transport for server-7-prod-us-east-1a:22222 (connect() failed: Connection refused) E0121 01:56:45.235440 71727 data-stream-sender.cc:278] channel send status: Couldn't open transport for server-7-prod-us-east-1a:22222 (connect() failed: Connection refused) E0121 01:56:45.483132 72134 data-stream-sender.cc:278] channel send status: Couldn't open transport for server-7-prod-us-east-1a:22222 (connect() failed: Connection refused) E0121 01:56:45.586616 75037 data-stream-sender.cc:278] channel send status: Couldn't open transport for server-7-prod-us-east-1a:22222 (connect() failed: Connection refused) E0121 01:56:45.910369 74878 data-stream-sender.cc:278] channel send status: Couldn't open transport for server-7-prod-us-east-1a:22222 (connect() failed: Connection refused) E0121 01:56:46.098247 72134 data-stream-sender.cc:278] channel send status: Couldn't open transport for server-7-prod-us-east-1a:22222 (connect() failed: Connection refused) E0121 01:56:46.121258 71153 data-stream-sender.cc:278] channel send status: Couldn't open transport for server-7-prod-us-east-1a:22222 (connect() failed: Connection refused) E0121 01:56:46.123798 71727 data-stream-sender.cc:278] channel send status: Couldn't open transport for server-7-prod-us-east-1a:22222 (connect() failed: Connection refused) E0121 01:56:46.817569 74843 data-stream-sender.cc:278] channel send status: Couldn't open transport for server-7-prod-us-east-1a:22222 (connect() failed: Connection refused) E0121 01:56:46.819041 74878 data-stream-sender.cc:278] channel send status: Couldn't open transport for server-7-prod-us-east-1a:22222 (connect() failed: Connection refused) E0121 01:56:46.819250 75037 data-stream-sender.cc:278] channel send status: Couldn't open transport for server-7-prod-us-east-1a:22222 (connect() failed: Connection refused) E0121 08:40:46.666100 2863 data-stream-sender.cc:278] channel send status: Couldn't open transport for server-8-prod-us-east-1a:22222 (connect() failed: Connection refused) E0121 08:40:46.676972 127538 data-stream-sender.cc:278] channel send status: Couldn't open transport for server-8-prod-us-east-1a:22222 (connect() failed: Connection refused) E0121 08:40:46.744699 32413 data-stream-sender.cc:278] channel send status: Couldn't open transport for server-8-prod-us-east-1a:22222 (connect() failed: Connection refused) E0121 08:40:46.747186 32200 data-stream-sender.cc:278] channel send status: Couldn't open transport for server-8-prod-us-east-1a:22222 (connect() failed: Connection refused) E0121 08:56:44.999950 88289 data-stream-sender.cc:278] channel send status: Couldn't open transport for server-8-prod-us-east-1a:22222 (connect() failed: Connection refused) E0121 09:07:26.185222 118904 data-stream-sender.cc:278] channel send status: TransmitData() to ux-reporting-engine-query-1-prod-us-east-1a:22222 failed: ECONNRESET E0121 09:07:26.194069 82768 data-stream-sender.cc:278] channel send status: Couldn't open transport for ux-reporting-engine-query-1-prod-us-east-1a:22222 (connect() failed: Connection refused) E0121 09:07:26.317500 82768 data-stream-sender.cc:278] channel send status: Couldn't open transport for ux-reporting-engine-query-1-prod-us-east-1a:22222 (connect() failed: Connection refused) E0121 09:07:27.159859 32200 data-stream-sender.cc:278] channel send status: Couldn't open transport for ux-reporting-engine-query-1-prod-us-east-1a:22222 (connect() failed: Connection refused) E0121 09:07:27.203852 32200 data-stream-sender.cc:278] channel send status: Couldn't open transport for server-8-prod-us-east-1a:22222 (connect() failed: Connection refused) E0121 09:07:27.474045 2863 data-stream-sender.cc:278] channel send status: Couldn't open transport for ux-reporting-engine-query-1-prod-us-east-1a:22222 (connect() failed: Connection refused) E0121 09:07:27.545581 2863 data-stream-sender.cc:278] channel send status: Couldn't open transport for server-8-prod-us-east-1a:22222 (connect() failed: Connection refused) E0121 09:07:27.581923 127538 data-stream-sender.cc:278] channel send status: Couldn't open transport for ux-reporting-engine-query-1-prod-us-east-1a:22222 (connect() failed: Connection refused) E0121 09:07:27.625087 127538 data-stream-sender.cc:278] channel send status: Couldn't open transport for server-8-prod-us-east-1a:22222 (connect() failed: Connection refused) E0121 09:07:27.884457 118904 data-stream-sender.cc:278] channel send status: TransmitData() to server-8-prod-us-east-1a:22222 failed: ECONNRESET E0121 09:07:27.906692 118904 data-stream-sender.cc:278] channel send status: TransmitData() to ux-reporting-engine-query-1-prod-us-east-1a:22222 failed: ECONNRESET E0121 09:12:29.026404 64029 data-stream-sender.cc:278] channel send status: Couldn't open transport for server-11-prod-us-east-1a:22222 (connect() failed: Connection refused) E0121 09:12:29.042855 63616 data-stream-sender.cc:278] channel send status: Couldn't open transport for server-11-prod-us-east-1a:22222 (connect() failed: Connection refused) E0121 09:12:29.044606 63756 data-stream-sender.cc:278] channel send status: Couldn't open transport for server-11-prod-us-east-1a:22222 (connect() failed: Connection refused) E0121 09:12:29.056113 63863 data-stream-sender.cc:278] channel send status: Couldn't open transport for server-11-prod-us-east-1a:22222 (connect() failed: Connection refused) E0121 09:12:29.091660 63689 data-stream-sender.cc:278] channel send status: Couldn't open transport for server-11-prod-us-east-1a:22222 (connect() failed: Connection refused) E0121 09:39:46.329826 849 data-stream-sender.cc:278] channel send status: Couldn't open transport for server-11-prod-us-east-1a:22222 (connect() failed: Connection refused) E0121 09:39:46.490386 127849 data-stream-sender.cc:278] channel send status: Couldn't open transport for server-11-prod-us-east-1a:22222 (connect() failed: Connection refused) E0121 09:39:46.912726 127849 data-stream-sender.cc:278] channel send status: Couldn't open transport for server-11-prod-us-east-1a:22222 (connect() failed: Connection refused) E0121 09:39:47.538194 849 data-stream-sender.cc:278] channel send status: Couldn't open transport for server-11-prod-us-east-1a:22222 (connect() failed: Connection refused) E0121 13:40:50.121517 49740 data-stream-sender.cc:278] channel send status: Couldn't open transport for ux-reporting-engine-query-1-prod-us-east-1a:22222 (connect() failed: Connection refused) E0121 13:40:50.135051 43893 data-stream-sender.cc:278] channel send status: Couldn't open transport for ux-reporting-engine-query-1-prod-us-east-1a:22222 (connect() failed: Connection refused) E0121 13:40:50.587116 45451 data-stream-sender.cc:278] channel send status: Couldn't open transport for ux-reporting-engine-query-1-prod-us-east-1a:22222 (connect() failed: Connection refused) E0121 13:40:50.757544 44525 data-stream-sender.cc:278] channel send status: Couldn't open transport for ux-reporting-engine-query-1-prod-us-east-1a:22222 (connect() failed: Connection refused) E0121 13:40:50.802755 44808 data-stream-sender.cc:278] channel send status: Couldn't open transport for ux-reporting-engine-query-1-prod-us-east-1a:22222 (connect() failed: Connection refused) E0121 13:40:51.449764 45451 data-stream-sender.cc:278] channel send status: Couldn't open transport for ux-reporting-engine-query-1-prod-us-east-1a:22222 (connect() failed: Connection refused) E0121 13:40:51.465267 44525 data-stream-sender.cc:278] channel send status: Couldn't open transport for ux-reporting-engine-query-1-prod-us-east-1a:22222 (connect() failed: Connection refused) E0121 13:40:51.474526 43893 data-stream-sender.cc:278] channel send status: Couldn't open transport for ux-reporting-engine-query-1-prod-us-east-1a:22222 (connect() failed: Connection refused) E0121 13:40:51.504053 44808 data-stream-sender.cc:278] channel send status: Couldn't open transport for ux-reporting-engine-query-1-prod-us-east-1a:22222 (connect() failed: Connection refused) E0121 14:01:49.790128 48456 data-stream-sender.cc:278] channel send status: Couldn't open transport for server-11-prod-us-east-1a:22222 (connect() failed: Connection refused) E0121 14:01:49.794749 8489 data-stream-sender.cc:278] channel send status: Couldn't open transport for server-11-prod-us-east-1a:22222 (connect() failed: Connection refused) E0121 14:01:49.899214 6653 data-stream-sender.cc:278] channel send status: Couldn't open transport for server-11-prod-us-east-1a:22222 (connect() failed: Connection refused) E0121 14:01:49.904781 48413 data-stream-sender.cc:278] channel send status: Couldn't open transport for server-11-prod-us-east-1a:22222 (connect() failed: Connection refused) E0121 14:01:50.278270 6653 data-stream-sender.cc:278] channel send status: Couldn't open transport for server-11-prod-us-east-1a:22222 (connect() failed: Connection refused) E0121 14:01:50.708433 8489 data-stream-sender.cc:278] channel send status: Couldn't open transport for server-11-prod-us-east-1a:22222 (connect() failed: Connection refused) E0121 14:01:52.789263 128491 data-stream-sender.cc:278] channel send status: Couldn't open transport for server-11-prod-us-east-1a:22222 (connect() failed: Connection refused) E0121 14:01:52.794026 128491 data-stream-sender.cc:278] channel send status: Couldn't open transport for ux-reporting-engine-query-1-prod-us-east-1a:22222 (connect() failed: Connection refused) E0121 14:03:48.985607 61629 data-stream-sender.cc:278] channel send status: Couldn't open transport for server-11-prod-us-east-1a:22222 (connect() failed: Connection refused) E0121 14:03:49.010754 61568 data-stream-sender.cc:278] channel send status: Couldn't open transport for server-11-prod-us-east-1a:22222 (connect() failed: Connection refused) E0121 14:39:40.617120 44373 data-stream-sender.cc:278] channel send status: Couldn't open transport for server-11-prod-us-east-1a:22222 (connect() failed: Connection refused) E0121 14:39:40.617125 43479 data-stream-sender.cc:278] channel send status: Couldn't open transport for server-11-prod-us-east-1a:22222 (connect() failed: Connection refused) E0121 14:39:40.671511 43598 data-stream-sender.cc:278] channel send status: Couldn't open transport for server-11-prod-us-east-1a:22222 (connect() failed: Connection refused) E0121 14:39:40.683009 43720 data-stream-sender.cc:278] channel send status: Couldn't open transport for server-11-prod-us-east-1a:22222 (connect() failed: Connection refused) E0121 14:39:41.017838 44178 data-stream-sender.cc:278] channel send status: Couldn't open transport for server-11-prod-us-east-1a:22222 (connect() failed: Connection refused) E0121 14:39:42.501684 43479 data-stream-sender.cc:278] channel send status: Couldn't open transport for server-11-prod-us-east-1a:22222 (connect() failed: Connection refused) E0121 14:39:42.522014 43720 data-stream-sender.cc:278] channel send status: Couldn't open transport for server-11-prod-us-east-1a:22222 (connect() failed: Connection refused) E0121 14:39:42.524489 43598 data-stream-sender.cc:278] channel send status: Couldn't open transport for server-11-prod-us-east-1a:22222 (connect() failed: Connection refused) E0121 14:42:14.467475 129083 data-stream-sender.cc:278] channel send status: Couldn't open transport for server-11-prod-us-east-1a:22222 (connect() failed: Connection refused) E0121 14:42:14.482939 65308 data-stream-sender.cc:278] channel send status: Couldn't open transport for server-11-prod-us-east-1a:22222 (connect() failed: Connection refused) E0121 14:42:14.663463 65165 data-stream-sender.cc:278] channel send status: Couldn't open transport for server-11-prod-us-east-1a:22222 (connect() failed: Connection refused) E0121 14:42:14.745048 129083 data-stream-sender.cc:278] channel send status: Couldn't open transport for ux-reporting-engine-query-1-prod-us-east-1a:22222 (connect() failed: Connection refused) E0121 14:42:14.748483 129083 data-stream-sender.cc:278] channel send status: Couldn't open transport for server-11-prod-us-east-1a:22222 (connect() failed: Connection refused) E0121 14:42:48.008329 68526 data-stream-sender.cc:278] channel send status: Couldn't open transport for ux-reporting-engine-query-1-prod-us-east-1a:22222 (connect() failed: Connection refused) E0121 14:42:48.009431 68489 data-stream-sender.cc:278] channel send status: Couldn't open transport for ux-reporting-engine-query-1-prod-us-east-1a:22222 (connect() failed: Connection refused) E0121 14:46:36.052253 97670 data-stream-sender.cc:278] channel send status: Couldn't open transport for server-11-prod-us-east-1a:22222 (connect() failed: Connection refused) E0121 14:46:50.626844 99839 data-stream-sender.cc:278] channel send status: Couldn't open transport for ux-reporting-engine-query-1-prod-us-east-1a:22222 (connect() failed: Connection refused) E0121 14:48:18.381661 108597 data-stream-sender.cc:278] channel send status: Couldn't open transport for server-11-prod-us-east-1a:22222 (connect() failed: Connection refused) E0121 14:48:18.400161 108693 data-stream-sender.cc:278] channel send status: Couldn't open transport for server-11-prod-us-east-1a:22222 (connect() failed: Connection refused) E0121 14:48:18.556179 111409 data-stream-sender.cc:278] channel send status: Couldn't open transport for server-11-prod-us-east-1a:22222 (connect() failed: Connection refused) E0121 14:48:18.683167 108693 data-stream-sender.cc:278] channel send status: Couldn't open transport for server-11-prod-us-east-1a:22222 (connect() failed: Connection refused) FSDataOutputStream#close error: org.apache.hadoop.ipc.RemoteException(org.apache.hadoop.hdfs.server.namenode.LeaseExpiredException): No lease on /user/hive/warehouse/realtime.db/dim_transaction_process_date_detail_extb/_impala_insert_staging/8d42df8618358f37_74a17a92577b0796/.8d42df8618358f37-74a17a92577b079f_1319351764_dir/process_date_key=2015-10-30/8d42df8618358f37-74a17a92577b079f_276460484_data.0.parq (inode 184083616): File does not exist. Holder DFSClient_NONMAPREDUCE_765389134_1 does not have any open files. at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.checkLease(FSNamesystem.java:3605) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.analyzeFileState(FSNamesystem.java:3402) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.getAdditionalBlock(FSNamesystem.java:3305) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.addBlock(NameNodeRpcServer.java:668) at org.apache.hadoop.hdfs.server.namenode.AuthorizationProviderProxyClientProtocol.addBlock(AuthorizationProviderProxyClientProtocol.java:212) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.addBlock(ClientNamenodeProtocolServerSideTranslatorPB.java:483) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine.java:619) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1060) at org.apache.hadoop.ipc.Server$Handler$1.run(Server.java:2044) at org.apache.hadoop.ipc.Server$Handler$1.run(Server.java:2040) at java.security.AccessController.doPrivileged(Native Method) at javax.security.auth.Subject.doAs(Subject.java:415) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1671) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:2038) at org.apache.hadoop.ipc.Client.call(Client.java:1468) at org.apache.hadoop.ipc.Client.call(Client.java:1399) at org.apache.hadoop.ipc.ProtobufRpcEngine$Invoker.invoke(ProtobufRpcEngine.java:232) at com.sun.proxy.$Proxy14.addBlock(Unknown Source) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.addBlock(ClientNamenodeProtocolTranslatorPB.java:399) at sun.reflect.GeneratedMethodAccessor15.invoke(Unknown Source) at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.lang.reflect.Method.invoke(Method.java:606) at org.apache.hadoop.io.retry.RetryInvocationHandler.invokeMethod(RetryInvocationHandler.java:187) at org.apache.hadoop.io.retry.RetryInvocationHandler.invoke(RetryInvocationHandler.java:102) at com.sun.proxy.$Proxy15.addBlock(Unknown Source) at org.apache.hadoop.hdfs.DFSOutputStream$DataStreamer.locateFollowingBlock(DFSOutputStream.java:1544) at org.apache.hadoop.hdfs.DFSOutputStream$DataStreamer.nextBlockOutputStream(DFSOutputStream.java:1361) at org.apache.hadoop.hdfs.DFSOutputStream$DataStreamer.run(DFSOutputStream.java:600) FSDataOutputStream#close error: org.apache.hadoop.ipc.RemoteException(org.apache.hadoop.hdfs.server.namenode.LeaseExpiredException): No lease on /user/hive/warehouse/realtime.db/dim_transaction_process_date_detail_extb/_impala_insert_staging/8d42df8618358f37_74a17a92577b0796/.8d42df8618358f37-74a17a92577b079f_115543783_dir/process_date_key=2013-11-05/8d42df8618358f37-74a17a92577b079f_424406236_data.0.parq (inode 184083610): File does not exist. Holder DFSClient_NONMAPREDUCE_765389134_1 does not have any open files. at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.checkLease(FSNamesystem.java:3605) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.analyzeFileState(FSNamesystem.java:3402) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.getAdditionalBlock(FSNamesystem.java:3258) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.addBlock(NameNodeRpcServer.java:668) at org.apache.hadoop.hdfs.server.namenode.AuthorizationProviderProxyClientProtocol.addBlock(AuthorizationProviderProxyClientProtocol.java:212) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.addBlock(ClientNamenodeProtocolServerSideTranslatorPB.java:483) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine.java:619) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1060) at org.apache.hadoop.ipc.Server$Handler$1.run(Server.java:2044) at org.apache.hadoop.ipc.Server$Handler$1.run(Server.java:2040) at java.security.AccessController.doPrivileged(Native Method) at javax.security.auth.Subject.doAs(Subject.java:415) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1671) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:2038) at org.apache.hadoop.ipc.Client.call(Client.java:1468) at org.apache.hadoop.ipc.Client.call(Client.java:1399) at org.apache.hadoop.ipc.ProtobufRpcEngine$Invoker.invoke(ProtobufRpcEngine.java:232) at com.sun.proxy.$Proxy14.addBlock(Unknown Source) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.addBlock(ClientNamenodeProtocolTranslatorPB.java:399) at sun.reflect.GeneratedMethodAccessor15.invoke(Unknown Source) at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.lang.reflect.Method.invoke(Method.java:606) at org.apache.hadoop.io.retry.RetryInvocationHandler.invokeMethod(RetryInvocationHandler.java:187) at org.apache.hadoop.io.retry.RetryInvocationHandler.invoke(RetryInvocationHandler.java:102) at com.sun.proxy.$Proxy15.addBlock(Unknown Source) at org.apache.hadoop.hdfs.DFSOutputStream$DataStreamer.locateFollowingBlock(DFSOutputStream.java:1544) at org.apache.hadoop.hdfs.DFSOutputStream$DataStreamer.nextBlockOutputStream(DFSOutputStream.java:1361) at org.apache.hadoop.hdfs.DFSOutputStream$DataStreamer.run(DFSOutputStream.java:600) FSDataOutputStream#close error: org.apache.hadoop.ipc.RemoteException(org.apache.hadoop.hdfs.server.namenode.LeaseExpiredException): No lease on /user/hive/warehouse/realtime.db/dim_transaction_process_date_detail_extb/_impala_insert_staging/8d42df8618358f37_74a17a92577b0796/.8d42df8618358f37-74a17a92577b079f_2110204190_dir/process_date_key=2015-11-14/8d42df8618358f37-74a17a92577b079f_1378189547_data.0.parq (inode 184083604): File does not exist. Holder DFSClient_NONMAPREDUCE_765389134_1 does not have any open files. at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.checkLease(FSNamesystem.java:3605) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.analyzeFileState(FSNamesystem.java:3402) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.getAdditionalBlock(FSNamesystem.java:3258) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.addBlock(NameNodeRpcServer.java:668) at org.apache.hadoop.hdfs.server.namenode.AuthorizationProviderProxyClientProtocol.addBlock(AuthorizationProviderProxyClientProtocol.java:212) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.addBlock(ClientNamenodeProtocolServerSideTranslatorPB.java:483) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine.java:619) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1060) at org.apache.hadoop.ipc.Server$Handler$1.run(Server.java:2044) at org.apache.hadoop.ipc.Server$Handler$1.run(Server.java:2040) at java.security.AccessController.doPrivileged(Native Method) at javax.security.auth.Subject.doAs(Subject.java:415) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1671) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:2038) at org.apache.hadoop.ipc.Client.call(Client.java:1468) at org.apache.hadoop.ipc.Client.call(Client.java:1399) at org.apache.hadoop.ipc.ProtobufRpcEngine$Invoker.invoke(ProtobufRpcEngine.java:232) at com.sun.proxy.$Proxy14.addBlock(Unknown Source) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.addBlock(ClientNamenodeProtocolTranslatorPB.java:399) at sun.reflect.GeneratedMethodAccessor15.invoke(Unknown Source) at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.lang.reflect.Method.invoke(Method.java:606) at org.apache.hadoop.io.retry.RetryInvocationHandler.invokeMethod(RetryInvocationHandler.java:187) at org.apache.hadoop.io.retry.RetryInvocationHandler.invoke(RetryInvocationHandler.java:102) at com.sun.proxy.$Proxy15.addBlock(Unknown Source) at org.apache.hadoop.hdfs.DFSOutputStream$DataStreamer.locateFollowingBlock(DFSOutputStream.java:1544) at org.apache.hadoop.hdfs.DFSOutputStream$DataStreamer.nextBlockOutputStream(DFSOutputStream.java:1361) at org.apache.hadoop.hdfs.DFSOutputStream$DataStreamer.run(DFSOutputStream.java:600) FSDataOutputStream#close error: org.apache.hadoop.ipc.RemoteException(org.apache.hadoop.hdfs.server.namenode.LeaseExpiredException): No lease on /user/hive/warehouse/realtime.db/dim_transaction_process_date_detail_extb/_impala_insert_staging/8d42df8618358f37_74a17a92577b0796/.8d42df8618358f37-74a17a92577b079f_65316281_dir/process_date_key=2015-09-08/8d42df8618358f37-74a17a92577b079f_1557428343_data.0.parq (inode 184083601): File does not exist. Holder DFSClient_NONMAPREDUCE_765389134_1 does not have any open files. at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.checkLease(FSNamesystem.java:3605) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.analyzeFileState(FSNamesystem.java:3402) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.getAdditionalBlock(FSNamesystem.java:3258) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.addBlock(NameNodeRpcServer.java:668) at org.apache.hadoop.hdfs.server.namenode.AuthorizationProviderProxyClientProtocol.addBlock(AuthorizationProviderProxyClientProtocol.java:212) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.addBlock(ClientNamenodeProtocolServerSideTranslatorPB.java:483) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine.java:619) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1060) at org.apache.hadoop.ipc.Server$Handler$1.run(Server.java:2044) at org.apache.hadoop.ipc.Server$Handler$1.run(Server.java:2040) at java.security.AccessController.doPrivileged(Native Method) at javax.security.auth.Subject.doAs(Subject.java:415) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1671) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:2038) at org.apache.hadoop.ipc.Client.call(Client.java:1468) at org.apache.hadoop.ipc.Client.call(Client.java:1399) at org.apache.hadoop.ipc.ProtobufRpcEngine$Invoker.invoke(ProtobufRpcEngine.java:232) at com.sun.proxy.$Proxy14.addBlock(Unknown Source) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.addBlock(ClientNamenodeProtocolTranslatorPB.java:399) at sun.reflect.GeneratedMethodAccessor15.invoke(Unknown Source) at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.lang.reflect.Method.invoke(Method.java:606) at org.apache.hadoop.io.retry.RetryInvocationHandler.invokeMethod(RetryInvocationHandler.java:187) at org.apache.hadoop.io.retry.RetryInvocationHandler.invoke(RetryInvocationHandler.java:102) at com.sun.proxy.$Proxy15.addBlock(Unknown Source) at org.apache.hadoop.hdfs.DFSOutputStream$DataStreamer.locateFollowingBlock(DFSOutputStream.java:1544) at org.apache.hadoop.hdfs.DFSOutputStream$DataStreamer.nextBlockOutputStream(DFSOutputStream.java:1361) at org.apache.hadoop.hdfs.DFSOutputStream$DataStreamer.run(DFSOutputStream.java:600) FSDataOutputStream#close error: org.apache.hadoop.ipc.RemoteException(org.apache.hadoop.hdfs.server.namenode.LeaseExpiredException): No lease on /user/hive/warehouse/realtime.db/dim_transaction_process_date_detail_extb/_impala_insert_staging/8d42df8618358f37_74a17a92577b0796/.8d42df8618358f37-74a17a92577b079f_356252115_dir/process_date_key=2015-10-04/8d42df8618358f37-74a17a92577b079f_1240370163_data.0.parq (inode 184083598): File does not exist. Holder DFSClient_NONMAPREDUCE_765389134_1 does not have any open files. at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.checkLease(FSNamesystem.java:3605) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.analyzeFileState(FSNamesystem.java:3402) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.getAdditionalBlock(FSNamesystem.java:3258) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.addBlock(NameNodeRpcServer.java:668) at org.apache.hadoop.hdfs.server.namenode.AuthorizationProviderProxyClientProtocol.addBlock(AuthorizationProviderProxyClientProtocol.java:212) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.addBlock(ClientNamenodeProtocolServerSideTranslatorPB.java:483) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine.java:619) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1060) at org.apache.hadoop.ipc.Server$Handler$1.run(Server.java:2044) at org.apache.hadoop.ipc.Server$Handler$1.run(Server.java:2040) at java.security.AccessController.doPrivileged(Native Method) at javax.security.auth.Subject.doAs(Subject.java:415) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1671) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:2038) at org.apache.hadoop.ipc.Client.call(Client.java:1468) at org.apache.hadoop.ipc.Client.call(Client.java:1399) at org.apache.hadoop.ipc.ProtobufRpcEngine$Invoker.invoke(ProtobufRpcEngine.java:232) at com.sun.proxy.$Proxy14.addBlock(Unknown Source) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.addBlock(ClientNamenodeProtocolTranslatorPB.java:399) at sun.reflect.GeneratedMethodAccessor15.invoke(Unknown Source) at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.lang.reflect.Method.invoke(Method.java:606) at org.apache.hadoop.io.retry.RetryInvocationHandler.invokeMethod(RetryInvocationHandler.java:187) at org.apache.hadoop.io.retry.RetryInvocationHandler.invoke(RetryInvocationHandler.java:102) at com.sun.proxy.$Proxy15.addBlock(Unknown Source) at org.apache.hadoop.hdfs.DFSOutputStream$DataStreamer.locateFollowingBlock(DFSOutputStream.java:1544) at org.apache.hadoop.hdfs.DFSOutputStream$DataStreamer.nextBlockOutputStream(DFSOutputStream.java:1361) at org.apache.hadoop.hdfs.DFSOutputStream$DataStreamer.run(DFSOutputStream.java:600) FSDataOutputStream#close error: org.apache.hadoop.ipc.RemoteException(org.apache.hadoop.hdfs.server.namenode.LeaseExpiredException): No lease on /user/hive/warehouse/realtime.db/dim_transaction_process_date_detail_extb/_impala_insert_staging/8d42df8618358f37_74a17a92577b0796/.8d42df8618358f37-74a17a92577b079f_1819326749_dir/process_date_key=2014-03-17/8d42df8618358f37-74a17a92577b079f_269427525_data.0.parq (inode 184083607): File does not exist. Holder DFSClient_NONMAPREDUCE_765389134_1 does not have any open files. at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.checkLease(FSNamesystem.java:3605) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.analyzeFileState(FSNamesystem.java:3402) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.getAdditionalBlock(FSNamesystem.java:3258) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.addBlock(NameNodeRpcServer.java:668) at org.apache.hadoop.hdfs.server.namenode.AuthorizationProviderProxyClientProtocol.addBlock(AuthorizationProviderProxyClientProtocol.java:212) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.addBlock(ClientNamenodeProtocolServerSideTranslatorPB.java:483) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine.java:619) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1060) at org.apache.hadoop.ipc.Server$Handler$1.run(Server.java:2044) at org.apache.hadoop.ipc.Server$Handler$1.run(Server.java:2040) at java.security.AccessController.doPrivileged(Native Method) at javax.security.auth.Subject.doAs(Subject.java:415) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1671) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:2038) at org.apache.hadoop.ipc.Client.call(Client.java:1468) at org.apache.hadoop.ipc.Client.call(Client.java:1399) at org.apache.hadoop.ipc.ProtobufRpcEngine$Invoker.invoke(ProtobufRpcEngine.java:232) at com.sun.proxy.$Proxy14.addBlock(Unknown Source) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.addBlock(ClientNamenodeProtocolTranslatorPB.java:399) at sun.reflect.GeneratedMethodAccessor15.invoke(Unknown Source) at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.lang.reflect.Method.invoke(Method.java:606) at org.apache.hadoop.io.retry.RetryInvocationHandler.invokeMethod(RetryInvocationHandler.java:187) at org.apache.hadoop.io.retry.RetryInvocationHandler.invoke(RetryInvocationHandler.java:102) at com.sun.proxy.$Proxy15.addBlock(Unknown Source) at org.apache.hadoop.hdfs.DFSOutputStream$DataStreamer.locateFollowingBlock(DFSOutputStream.java:1544) at org.apache.hadoop.hdfs.DFSOutputStream$DataStreamer.nextBlockOutputStream(DFSOutputStream.java:1361) at org.apache.hadoop.hdfs.DFSOutputStream$DataStreamer.run(DFSOutputStream.java:600) FSDataOutputStream#close error: org.apache.hadoop.ipc.RemoteException(org.apache.hadoop.hdfs.server.namenode.LeaseExpiredException): No lease on /user/hive/warehouse/realtime.db/dim_transaction_process_date_detail_extb/_impala_insert_staging/8d42df8618358f37_74a17a92577b0796/.8d42df8618358f37-74a17a92577b079f_341162310_dir/process_date_key=2013-11-14/8d42df8618358f37-74a17a92577b079f_1062658252_data.0.parq (inode 184083595): File does not exist. Holder DFSClient_NONMAPREDUCE_765389134_1 does not have any open files. at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.checkLease(FSNamesystem.java:3605) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.analyzeFileState(FSNamesystem.java:3402) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.getAdditionalBlock(FSNamesystem.java:3258) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.addBlock(NameNodeRpcServer.java:668) at org.apache.hadoop.hdfs.server.namenode.AuthorizationProviderProxyClientProtocol.addBlock(AuthorizationProviderProxyClientProtocol.java:212) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.addBlock(ClientNamenodeProtocolServerSideTranslatorPB.java:483) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine.java:619) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1060) at org.apache.hadoop.ipc.Server$Handler$1.run(Server.java:2044) at org.apache.hadoop.ipc.Server$Handler$1.run(Server.java:2040) at java.security.AccessController.doPrivileged(Native Method) at javax.security.auth.Subject.doAs(Subject.java:415) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1671) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:2038) at org.apache.hadoop.ipc.Client.call(Client.java:1468) at org.apache.hadoop.ipc.Client.call(Client.java:1399) at org.apache.hadoop.ipc.ProtobufRpcEngine$Invoker.invoke(ProtobufRpcEngine.java:232) at com.sun.proxy.$Proxy14.addBlock(Unknown Source) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.addBlock(ClientNamenodeProtocolTranslatorPB.java:399) at sun.reflect.GeneratedMethodAccessor15.invoke(Unknown Source) at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.lang.reflect.Method.invoke(Method.java:606) at org.apache.hadoop.io.retry.RetryInvocationHandler.invokeMethod(RetryInvocationHandler.java:187) at org.apache.hadoop.io.retry.RetryInvocationHandler.invoke(RetryInvocationHandler.java:102) at com.sun.proxy.$Proxy15.addBlock(Unknown Source) at org.apache.hadoop.hdfs.DFSOutputStream$DataStreamer.locateFollowingBlock(DFSOutputStream.java:1544) at org.apache.hadoop.hdfs.DFSOutputStream$DataStreamer.nextBlockOutputStream(DFSOutputStream.java:1361) at org.apache.hadoop.hdfs.DFSOutputStream$DataStreamer.run(DFSOutputStream.java:600) FSDataOutputStream#close error: org.apache.hadoop.ipc.RemoteException(org.apache.hadoop.hdfs.server.namenode.LeaseExpiredException): No lease on /user/hive/warehouse/realtime.db/dim_transaction_process_date_detail_extb/_impala_insert_staging/8d42df8618358f37_74a17a92577b0796/.8d42df8618358f37-74a17a92577b079f_116050784_dir/process_date_key=2014-07-28/8d42df8618358f37-74a17a92577b079f_588912658_data.0.parq (inode 184083592): File does not exist. Holder DFSClient_NONMAPREDUCE_765389134_1 does not have any open files. at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.checkLease(FSNamesystem.java:3605) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.analyzeFileState(FSNamesystem.java:3402) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.getAdditionalBlock(FSNamesystem.java:3258) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.addBlock(NameNodeRpcServer.java:668) at org.apache.hadoop.hdfs.server.namenode.AuthorizationProviderProxyClientProtocol.addBlock(AuthorizationProviderProxyClientProtocol.java:212) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.addBlock(ClientNamenodeProtocolServerSideTranslatorPB.java:483) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine.java:619) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1060) at org.apache.hadoop.ipc.Server$Handler$1.run(Server.java:2044) at org.apache.hadoop.ipc.Server$Handler$1.run(Server.java:2040) at java.security.AccessController.doPrivileged(Native Method) at javax.security.auth.Subject.doAs(Subject.java:415) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1671) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:2038) at org.apache.hadoop.ipc.Client.call(Client.java:1468) at org.apache.hadoop.ipc.Client.call(Client.java:1399) at org.apache.hadoop.ipc.ProtobufRpcEngine$Invoker.invoke(ProtobufRpcEngine.java:232) at com.sun.proxy.$Proxy14.addBlock(Unknown Source) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.addBlock(ClientNamenodeProtocolTranslatorPB.java:399) at sun.reflect.GeneratedMethodAccessor15.invoke(Unknown Source) at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.lang.reflect.Method.invoke(Method.java:606) at org.apache.hadoop.io.retry.RetryInvocationHandler.invokeMethod(RetryInvocationHandler.java:187) at org.apache.hadoop.io.retry.RetryInvocationHandler.invoke(RetryInvocationHandler.java:102) at com.sun.proxy.$Proxy15.addBlock(Unknown Source) at org.apache.hadoop.hdfs.DFSOutputStream$DataStreamer.locateFollowingBlock(DFSOutputStream.java:1544) at org.apache.hadoop.hdfs.DFSOutputStream$DataStreamer.nextBlockOutputStream(DFSOutputStream.java:1361) at org.apache.hadoop.hdfs.DFSOutputStream$DataStreamer.run(DFSOutputStream.java:600) FSDataOutputStream#close error: org.apache.hadoop.ipc.RemoteException(org.apache.hadoop.hdfs.server.namenode.LeaseExpiredException): No lease on /user/hive/warehouse/realtime.db/dim_transaction_process_date_detail_extb/_impala_insert_staging/8d42df8618358f37_74a17a92577b0796/.8d42df8618358f37-74a17a92577b079f_388786510_dir/process_date_key=2015-12-17/8d42df8618358f37-74a17a92577b079f_49378728_data.0.parq (inode 184083573): File does not exist. Holder DFSClient_NONMAPREDUCE_765389134_1 does not have any open files. at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.checkLease(FSNamesystem.java:3605) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.analyzeFileState(FSNamesystem.java:3402) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.getAdditionalBlock(FSNamesystem.java:3258) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.addBlock(NameNodeRpcServer.java:668) at org.apache.hadoop.hdfs.server.namenode.AuthorizationProviderProxyClientProtocol.addBlock(AuthorizationProviderProxyClientProtocol.java:212) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.addBlock(ClientNamenodeProtocolServerSideTranslatorPB.java:483) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine.java:619) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1060) at org.apache.hadoop.ipc.Server$Handler$1.run(Server.java:2044) at org.apache.hadoop.ipc.Server$Handler$1.run(Server.java:2040) at java.security.AccessController.doPrivileged(Native Method) at javax.security.auth.Subject.doAs(Subject.java:415) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1671) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:2038) at org.apache.hadoop.ipc.Client.call(Client.java:1468) at org.apache.hadoop.ipc.Client.call(Client.java:1399) at org.apache.hadoop.ipc.ProtobufRpcEngine$Invoker.invoke(ProtobufRpcEngine.java:232) at com.sun.proxy.$Proxy14.addBlock(Unknown Source) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.addBlock(ClientNamenodeProtocolTranslatorPB.java:399) at sun.reflect.GeneratedMethodAccessor15.invoke(Unknown Source) at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.lang.reflect.Method.invoke(Method.java:606) at org.apache.hadoop.io.retry.RetryInvocationHandler.invokeMethod(RetryInvocationHandler.java:187) at org.apache.hadoop.io.retry.RetryInvocationHandler.invoke(RetryInvocationHandler.java:102) at com.sun.proxy.$Proxy15.addBlock(Unknown Source) at org.apache.hadoop.hdfs.DFSOutputStream$DataStreamer.locateFollowingBlock(DFSOutputStream.java:1544) at org.apache.hadoop.hdfs.DFSOutputStream$DataStreamer.nextBlockOutputStream(DFSOutputStream.java:1361) at org.apache.hadoop.hdfs.DFSOutputStream$DataStreamer.run(DFSOutputStream.java:600) FSDataOutputStream#close error: org.apache.hadoop.ipc.RemoteException(org.apache.hadoop.hdfs.server.namenode.LeaseExpiredException): No lease on /user/hive/warehouse/realtime.db/dim_transaction_process_date_detail_extb/_impala_insert_staging/8d42df8618358f37_74a17a92577b0796/.8d42df8618358f37-74a17a92577b079f_1215220473_dir/process_date_key=2015-12-16/8d42df8618358f37-74a17a92577b079f_392999489_data.0.parq (inode 184083567): File does not exist. Holder DFSClient_NONMAPREDUCE_765389134_1 does not have any open files. at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.checkLease(FSNamesystem.java:3605) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.analyzeFileState(FSNamesystem.java:3402) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.getAdditionalBlock(FSNamesystem.java:3258) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.addBlock(NameNodeRpcServer.java:668) at org.apache.hadoop.hdfs.server.namenode.AuthorizationProviderProxyClientProtocol.addBlock(AuthorizationProviderProxyClientProtocol.java:212) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.addBlock(ClientNamenodeProtocolServerSideTranslatorPB.java:483) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine.java:619) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1060) at org.apache.hadoop.ipc.Server$Handler$1.run(Server.java:2044) at org.apache.hadoop.ipc.Server$Handler$1.run(Server.java:2040) at java.security.AccessController.doPrivileged(Native Method) at javax.security.auth.Subject.doAs(Subject.java:415) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1671) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:2038) at org.apache.hadoop.ipc.Client.call(Client.java:1468) at org.apache.hadoop.ipc.Client.call(Client.java:1399) at org.apache.hadoop.ipc.ProtobufRpcEngine$Invoker.invoke(ProtobufRpcEngine.java:232) at com.sun.proxy.$Proxy14.addBlock(Unknown Source) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.addBlock(ClientNamenodeProtocolTranslatorPB.java:399) at sun.reflect.GeneratedMethodAccessor15.invoke(Unknown Source) at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.lang.reflect.Method.invoke(Method.java:606) at org.apache.hadoop.io.retry.RetryInvocationHandler.invokeMethod(RetryInvocationHandler.java:187) at org.apache.hadoop.io.retry.RetryInvocationHandler.invoke(RetryInvocationHandler.java:102) at com.sun.proxy.$Proxy15.addBlock(Unknown Source) at org.apache.hadoop.hdfs.DFSOutputStream$DataStreamer.locateFollowingBlock(DFSOutputStream.java:1544) at org.apache.hadoop.hdfs.DFSOutputStream$DataStreamer.nextBlockOutputStream(DFSOutputStream.java:1361) at org.apache.hadoop.hdfs.DFSOutputStream$DataStreamer.run(DFSOutputStream.java:600) FSDataOutputStream#close error: org.apache.hadoop.ipc.RemoteException(org.apache.hadoop.hdfs.server.namenode.LeaseExpiredException): No lease on /user/hive/warehouse/realtime.db/dim_transaction_process_date_detail_extb/_impala_insert_staging/8d42df8618358f37_74a17a92577b0796/.8d42df8618358f37-74a17a92577b079f_2046430086_dir/process_date_key=2015-12-27/8d42df8618358f37-74a17a92577b079f_364074013_data.0.parq (inode 184083564): File does not exist. Holder DFSClient_NONMAPREDUCE_765389134_1 does not have any open files. at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.checkLease(FSNamesystem.java:3605) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.analyzeFileState(FSNamesystem.java:3402) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.getAdditionalBlock(FSNamesystem.java:3258) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.addBlock(NameNodeRpcServer.java:668) at org.apache.hadoop.hdfs.server.namenode.AuthorizationProviderProxyClientProtocol.addBlock(AuthorizationProviderProxyClientProtocol.java:212) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.addBlock(ClientNamenodeProtocolServerSideTranslatorPB.java:483) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine.java:619) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1060) at org.apache.hadoop.ipc.Server$Handler$1.run(Server.java:2044) at org.apache.hadoop.ipc.Server$Handler$1.run(Server.java:2040) at java.security.AccessController.doPrivileged(Native Method) at javax.security.auth.Subject.doAs(Subject.java:415) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1671) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:2038) at org.apache.hadoop.ipc.Client.call(Client.java:1468) at org.apache.hadoop.ipc.Client.call(Client.java:1399) at org.apache.hadoop.ipc.ProtobufRpcEngine$Invoker.invoke(ProtobufRpcEngine.java:232) at com.sun.proxy.$Proxy14.addBlock(Unknown Source) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.addBlock(ClientNamenodeProtocolTranslatorPB.java:399) at sun.reflect.GeneratedMethodAccessor15.invoke(Unknown Source) at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.lang.reflect.Method.invoke(Method.java:606) at org.apache.hadoop.io.retry.RetryInvocationHandler.invokeMethod(RetryInvocationHandler.java:187) at org.apache.hadoop.io.retry.RetryInvocationHandler.invoke(RetryInvocationHandler.java:102) at com.sun.proxy.$Proxy15.addBlock(Unknown Source) at org.apache.hadoop.hdfs.DFSOutputStream$DataStreamer.locateFollowingBlock(DFSOutputStream.java:1544) at org.apache.hadoop.hdfs.DFSOutputStream$DataStreamer.nextBlockOutputStream(DFSOutputStream.java:1361) at org.apache.hadoop.hdfs.DFSOutputStream$DataStreamer.run(DFSOutputStream.java:600) FSDataOutputStream#close error: org.apache.hadoop.ipc.RemoteException(org.apache.hadoop.hdfs.server.namenode.LeaseExpiredException): No lease on /user/hive/warehouse/realtime.db/dim_transaction_process_date_detail_extb/_impala_insert_staging/8d42df8618358f37_74a17a92577b0796/.8d42df8618358f37-74a17a92577b079f_1587877885_dir/process_date_key=2015-01-28/8d42df8618358f37-74a17a92577b079f_789202164_data.0.parq (inode 184083561): File does not exist. Holder DFSClient_NONMAPREDUCE_765389134_1 does not have any open files. at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.checkLease(FSNamesystem.java:3605) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.analyzeFileState(FSNamesystem.java:3402) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.getAdditionalBlock(FSNamesystem.java:3258) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.addBlock(NameNodeRpcServer.java:668) at org.apache.hadoop.hdfs.server.namenode.AuthorizationProviderProxyClientProtocol.addBlock(AuthorizationProviderProxyClientProtocol.java:212) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.addBlock(ClientNamenodeProtocolServerSideTranslatorPB.java:483) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine.java:619) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1060) at org.apache.hadoop.ipc.Server$Handler$1.run(Server.java:2044) at org.apache.hadoop.ipc.Server$Handler$1.run(Server.java:2040) at java.security.AccessController.doPrivileged(Native Method) at javax.security.auth.Subject.doAs(Subject.java:415) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1671) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:2038) at org.apache.hadoop.ipc.Client.call(Client.java:1468) at org.apache.hadoop.ipc.Client.call(Client.java:1399) at org.apache.hadoop.ipc.ProtobufRpcEngine$Invoker.invoke(ProtobufRpcEngine.java:232) at com.sun.proxy.$Proxy14.addBlock(Unknown Source) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.addBlock(ClientNamenodeProtocolTranslatorPB.java:399) at sun.reflect.GeneratedMethodAccessor15.invoke(Unknown Source) at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.lang.reflect.Method.invoke(Method.java:606) at org.apache.hadoop.io.retry.RetryInvocationHandler.invokeMethod(RetryInvocationHandler.java:187) at org.apache.hadoop.io.retry.RetryInvocationHandler.invoke(RetryInvocationHandler.java:102) at com.sun.proxy.$Proxy15.addBlock(Unknown Source) at org.apache.hadoop.hdfs.DFSOutputStream$DataStreamer.locateFollowingBlock(DFSOutputStream.java:1544) at org.apache.hadoop.hdfs.DFSOutputStream$DataStreamer.nextBlockOutputStream(DFSOutputStream.java:1361) at org.apache.hadoop.hdfs.DFSOutputStream$DataStreamer.run(DFSOutputStream.java:600) FSDataOutputStream#close error: org.apache.hadoop.ipc.RemoteException(org.apache.hadoop.hdfs.server.namenode.LeaseExpiredException): No lease on /user/hive/warehouse/realtime.db/dim_transaction_process_date_detail_extb/_impala_insert_staging/8d42df8618358f37_74a17a92577b0796/.8d42df8618358f37-74a17a92577b079f_1088542885_dir/process_date_key=2015-12-09/8d42df8618358f37-74a17a92577b079f_1325708228_data.0.parq (inode 184083558): File dE0121 14:48:18.856155 112452 data-stream-sender.cc:278] channel send status: Couldn't open transport for server-11-prod-us-east-1a:22222 (connect() failed: Connection refused) E0121 14:48:18.962314 111409 data-stream-sender.cc:278] channel send status: Couldn't open transport for server-11-prod-us-east-1a:22222 (connect() failed: Connection refused) E0121 14:48:19.166868 108788 data-stream-sender.cc:278] channel send status: Couldn't open transport for server-11-prod-us-east-1a:22222 (connect() failed: Connection refused) E0121 14:48:19.718487 108597 data-stream-sender.cc:278] channel send status: Couldn't open transport for server-11-prod-us-east-1a:22222 (connect() failed: Connection refused) E0121 14:48:19.726614 108788 data-stream-sender.cc:278] channel send status: Couldn't open transport for server-11-prod-us-east-1a:22222 (connect() failed: Connection refused) E0121 14:48:20.092802 111601 data-stream-sender.cc:278] channel send status: Couldn't open transport for server-11-prod-us-east-1a:22222 (connect() failed: Connection refused) E0121 14:48:52.957592 117105 data-stream-sender.cc:278] channel send status: Couldn't open transport for server-11-prod-us-east-1a:22222 (connect() failed: Connection refused) E0121 14:48:52.959257 117030 data-stream-sender.cc:278] channel send status: Couldn't open transport for server-11-prod-us-east-1a:22222 (connect() failed: Connection refused) E0121 14:48:52.971688 116672 data-stream-sender.cc:278] channel send status: Couldn't open transport for server-11-prod-us-east-1a:22222 (connect() failed: Connection refused) E0121 14:48:53.409940 116539 data-stream-sender.cc:278] channel send status: Couldn't open transport for server-11-prod-us-east-1a:22222 (connect() failed: Connection refused) E0121 14:48:53.569957 117105 data-stream-sender.cc:278] channel send status: Couldn't open transport for server-11-prod-us-east-1a:22222 (connect() failed: Connection refused) E0121 14:49:25.504021 119161 data-stream-sender.cc:278] channel send status: Couldn't open transport for server-11-prod-us-east-1a:22222 (connect() failed: Connection refused) E0121 14:51:04.701807 7776 data-stream-sender.cc:278] channel send status: Couldn't open transport for server-9-prod-us-east-1a:22222 (connect() failed: Connection refused) E0121 14:51:04.715389 125804 data-stream-sender.cc:278] channel send status: Couldn't open transport for server-9-prod-us-east-1a:22222 (connect() failed: Connection refused) E0121 14:51:04.784723 2725 data-stream-sender.cc:278] channel send status: Couldn't open transport for server-9-prod-us-east-1a:22222 (connect() failed: Connection refused) E0121 14:51:04.827147 7279 data-stream-sender.cc:278] channel send status: Couldn't open transport for server-9-prod-us-east-1a:22222 (connect() failed: Connection refused) E0121 14:51:04.832963 4600 data-stream-sender.cc:278] channel send status: Couldn't open transport for server-9-prod-us-east-1a:22222 (connect() failed: Connection refused) E0121 14:51:05.437568 4600 data-stream-sender.cc:278] channel send status: Couldn't open transport for server-9-prod-us-east-1a:22222 (connect() failed: Connection refused) E0121 14:51:05.473280 2725 data-stream-sender.cc:278] channel send status: Couldn't open transport for server-9-prod-us-east-1a:22222 (connect() failed: Connection refused) E0121 14:51:05.584221 125804 data-stream-sender.cc:278] channel send status: Couldn't open transport for server-9-prod-us-east-1a:22222 (connect() failed: Connection refused) E0121 14:51:05.814991 7102 data-stream-sender.cc:278] channel send status: Couldn't open transport for server-9-prod-us-east-1a:22222 (connect() failed: Connection refused) E0121 17:01:41.591555 73527 data-stream-sender.cc:278] channel send status: Couldn't open transport for server-2-prod-us-east-1a:22222 (connect() failed: Connection refused) E0121 17:01:41.613173 96483 data-stream-sender.cc:278] channel send status: Couldn't open transport for server-2-prod-us-east-1a:22222 (connect() failed: Connection refused) E0121 17:01:41.629390 82043 data-stream-sender.cc:278] channel send status: Couldn't open transport for server-2-prod-us-east-1a:22222 (connect() failed: Connection refused) E0121 17:01:41.673635 96495 data-stream-sender.cc:278] channel send status: Couldn't open transport for server-2-prod-us-east-1a:22222 (connect() failed: Connection refused) E0121 17:01:41.694998 94174 data-stream-sender.cc:278] channel send status: Couldn't open transport for server-2-prod-us-east-1a:22222 (connect() failed: Connection refused) E0121 17:01:41.706779 96460 data-stream-sender.cc:278] channel send status: Couldn't open transport for server-2-prod-us-east-1a:22222 (connect() failed: Connection refused) E0121 17:01:41.714074 96439 data-stream-sender.cc:278] channel send status: Couldn't open transport for server-2-prod-us-east-1a:22222 (connect() failed: Connection refused) E0121 17:01:41.883457 73663 data-stream-sender.cc:278] channel send status: Couldn't open transport for server-2-prod-us-east-1a:22222 (connect() failed: Connection refused) E0121 17:01:41.956440 96495 data-stream-sender.cc:278] channel send status: Couldn't open transport for server-2-prod-us-east-1a:22222 (connect() failed: Connection refused) E0121 17:01:42.127228 96460 data-stream-sender.cc:278] channel send status: Couldn't open transport for server-2-prod-us-east-1a:22222 (connect() failed: Connection refused) E0121 17:01:42.205953 82811 data-stream-sender.cc:278] channel send status: Couldn't open transport for server-2-prod-us-east-1a:22222 (connect() failed: Connection refused) E0121 17:01:42.451828 94174 data-stream-sender.cc:278] channel send status: Couldn't open transport for server-2-prod-us-east-1a:22222 (connect() failed: Connection refused) E0121 17:01:42.677495 96439 data-stream-sender.cc:278] channel send status: Couldn't open transport for server-2-prod-us-east-1a:22222 (connect() failed: Connection refused) E0121 17:01:42.784942 73663 data-stream-sender.cc:278] channel send status: Couldn't open transport for server-2-prod-us-east-1a:22222 (connect() failed: Connection refused) E0121 17:01:42.830080 73527 data-stream-sender.cc:278] channel send status: Couldn't open transport for server-2-prod-us-east-1a:22222 (connect() failed: Connection refused) E0121 17:01:43.137310 73363 data-stream-sender.cc:278] channel send status: Couldn't open transport for server-2-prod-us-east-1a:22222 (connect() failed: Connection refused) E0121 17:01:43.347831 82811 data-stream-sender.cc:278] channel send status: Couldn't open transport for server-2-prod-us-east-1a:22222 (connect() failed: Connection refused) E0121 17:01:43.372174 82043 data-stream-sender.cc:278] channel send status: Couldn't open transport for server-2-prod-us-east-1a:22222 (connect() failed: Connection refused) E0121 17:01:43.596946 81879 data-stream-sender.cc:278] channel send status: Couldn't open transport for server-2-prod-us-east-1a:22222 (connect() failed: Connection refused) oes not exist. Holder DFSClient_NONMAPREDUCE_765389134_1 does not have any open files. at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.checkLease(FSNamesystem.java:3605) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.analyzeFileState(FSNamesystem.java:3402) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.getAdditionalBlock(FSNamesystem.java:3258) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.addBlock(NameNodeRpcServer.java:668) at org.apache.hadoop.hdfs.server.namenode.AuthorizationProviderProxyClientProtocol.addBlock(AuthorizationProviderProxyClientProtocol.java:212) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.addBlock(ClientNamenodeProtocolServerSideTranslatorPB.java:483) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine.java:619) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1060) at org.apache.hadoop.ipc.Server$Handler$1.run(Server.java:2044) at org.apache.hadoop.ipc.Server$Handler$1.run(Server.java:2040) at java.security.AccessController.doPrivileged(Native Method) at javax.security.auth.Subject.doAs(Subject.java:415) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1671) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:2038) at org.apache.hadoop.ipc.Client.call(Client.java:1468) at org.apache.hadoop.ipc.Client.call(Client.java:1399) at org.apache.hadoop.ipc.ProtobufRpcEngine$Invoker.invoke(ProtobufRpcEngine.java:232) at com.sun.proxy.$Proxy14.addBlock(Unknown Source) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.addBlock(ClientNamenodeProtocolTranslatorPB.java:399) at sun.reflect.GeneratedMethodAccessor15.invoke(Unknown Source) at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.lang.reflect.Method.invoke(Method.java:606) at org.apache.hadoop.io.retry.RetryInvocationHandler.invokeMethod(RetryInvocationHandler.java:187) at org.apache.hadoop.io.retry.RetryInvocationHandler.invoke(RetryInvocationHandler.java:102) at com.sun.proxy.$Proxy15.addBlock(Unknown Source) at org.apache.hadoop.hdfs.DFSOutputStream$DataStreamer.locateFollowingBlock(DFSOutputStream.java:1544) at org.apache.hadoop.hdfs.DFSOutputStream$DataStreamer.nextBlockOutputStream(DFSOutputStream.java:1361) at org.apache.hadoop.hdfs.DFSOutputStream$DataStreamer.run(DFSOutputStream.java:600)