2021-05-07T19:42:02,586 DEBUG [Tez session start thread([])]: retry.RetryInvocationHandler (RetryInvocationHandler.java:log(400)) - Exception while invoking ApplicationClientProtocolPBClientImpl.getNewApplication over rm1. Trying to failover immediately. java.io.IOException: Failed on local exception: java.io.InterruptedIOException: Interrupted: action=RetryAction(action=RETRY, delayMillis=1000, reason=retries get failed due to exceeded maximum allowed retries number: 4), retry policy=RetryUpToMaximumCountWithFixedSleep(maxRetries=4, sleepTime=1000 MILLISECONDS); Host Details : local host is: "i-048b6b85d1eb159db.inst.aws.us-east-1.prod.musta.ch/172.21.14.169"; destination host is: "ip-172-21-173-93.ec2.internal":8032; at org.apache.hadoop.net.NetUtils.wrapException(NetUtils.java:785) ~[hadoop-common-2.8.5-amzn-3.jar:?] at org.apache.hadoop.ipc.Client.getRpcResponse(Client.java:1493) ~[hadoop-common-2.8.5-amzn-3.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1435) ~[hadoop-common-2.8.5-amzn-3.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1345) ~[hadoop-common-2.8.5-amzn-3.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine$Invoker.invoke(ProtobufRpcEngine.java:227) ~[hadoop-common-2.8.5-amzn-3.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine$Invoker.invoke(ProtobufRpcEngine.java:116) ~[hadoop-common-2.8.5-amzn-3.jar:?] at com.sun.proxy.$Proxy38.getNewApplication(Unknown Source) ~[?:?] at org.apache.hadoop.yarn.api.impl.pb.client.ApplicationClientProtocolPBClientImpl.getNewApplication(ApplicationClientProtocolPBClientImpl.java:258) ~[hadoop-yarn-common-2.8.5-amzn-3.jar:?] at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:1.8.0_25] at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) ~[?:1.8.0_25] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_25] at java.lang.reflect.Method.invoke(Method.java:483) ~[?:1.8.0_25] at org.apache.hadoop.io.retry.RetryInvocationHandler.invokeMethod(RetryInvocationHandler.java:409) ~[hadoop-common-2.8.5-amzn-3.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeMethod(RetryInvocationHandler.java:163) ~[hadoop-common-2.8.5-amzn-3.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invoke(RetryInvocationHandler.java:155) ~[hadoop-common-2.8.5-amzn-3.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeOnce(RetryInvocationHandler.java:95) ~[hadoop-common-2.8.5-amzn-3.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler.invoke(RetryInvocationHandler.java:346) ~[hadoop-common-2.8.5-amzn-3.jar:?] at com.sun.proxy.$Proxy39.getNewApplication(Unknown Source) ~[?:?] at org.apache.hadoop.yarn.client.api.impl.YarnClientImpl.getNewApplication(YarnClientImpl.java:224) ~[hadoop-yarn-client-2.8.5-amzn-3.jar:?] at org.apache.hadoop.yarn.client.api.impl.YarnClientImpl.createApplication(YarnClientImpl.java:232) ~[hadoop-yarn-client-2.8.5-amzn-3.jar:?] at org.apache.tez.client.TezYarnClient.createApplication(TezYarnClient.java:71) ~[tez-api-0.9.1.jar:0.9.1] at org.apache.tez.client.TezClient.createApplication(TezClient.java:1043) ~[tez-api-0.9.1.jar:0.9.1] at org.apache.tez.client.TezClient.start(TezClient.java:420) ~[tez-api-0.9.1.jar:0.9.1] at org.apache.hadoop.hive.ql.exec.tez.TezSessionState.startSessionAndContainers(TezSessionState.java:376) ~[hive-exec-2.3.4.41.jar:2.3.4.41] at org.apache.hadoop.hive.ql.exec.tez.TezSessionState.access$000(TezSessionState.java:87) ~[hive-exec-2.3.4.41.jar:2.3.4.41] at org.apache.hadoop.hive.ql.exec.tez.TezSessionState$1.call(TezSessionState.java:330) ~[hive-exec-2.3.4.41.jar:2.3.4.41] at org.apache.hadoop.hive.ql.exec.tez.TezSessionState$1.call(TezSessionState.java:326) ~[hive-exec-2.3.4.41.jar:2.3.4.41] at java.util.concurrent.FutureTask.run(FutureTask.java:266) ~[?:1.8.0_25] at java.lang.Thread.run(Thread.java:745) [?:1.8.0_25] Caused by: java.io.InterruptedIOException: Interrupted: action=RetryAction(action=RETRY, delayMillis=1000, reason=retries get failed due to exceeded maximum allowed retries number: 4), retry policy=RetryUpToMaximumCountWithFixedSleep(maxRetries=4, sleepTime=1000 MILLISECONDS) at org.apache.hadoop.ipc.Client$Connection.handleConnectionFailure(Client.java:935) ~[hadoop-common-2.8.5-amzn-3.jar:?] at org.apache.hadoop.ipc.Client$Connection.setupConnection(Client.java:701) ~[hadoop-common-2.8.5-amzn-3.jar:?] at org.apache.hadoop.ipc.Client$Connection.setupIOstreams(Client.java:788) ~[hadoop-common-2.8.5-amzn-3.jar:?] at org.apache.hadoop.ipc.Client$Connection.access$3500(Client.java:410) ~[hadoop-common-2.8.5-amzn-3.jar:?] at org.apache.hadoop.ipc.Client.getConnection(Client.java:1550) ~[hadoop-common-2.8.5-amzn-3.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1381) ~[hadoop-common-2.8.5-amzn-3.jar:?] ... 26 more Caused by: java.lang.InterruptedException: sleep interrupted at java.lang.Thread.sleep(Native Method) ~[?:1.8.0_25] at org.apache.hadoop.ipc.Client$Connection.handleConnectionFailure(Client.java:933) ~[hadoop-common-2.8.5-amzn-3.jar:?] at org.apache.hadoop.ipc.Client$Connection.setupConnection(Client.java:701) ~[hadoop-common-2.8.5-amzn-3.jar:?] at org.apache.hadoop.ipc.Client$Connection.setupIOstreams(Client.java:788) ~[hadoop-common-2.8.5-amzn-3.jar:?] at org.apache.hadoop.ipc.Client$Connection.access$3500(Client.java:410) ~[hadoop-common-2.8.5-amzn-3.jar:?] at org.apache.hadoop.ipc.Client.getConnection(Client.java:1550) ~[hadoop-common-2.8.5-amzn-3.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1381) ~[hadoop-common-2.8.5-amzn-3.jar:?] ... 26 more 2021-05-07T19:42:02,586 INFO [Tez session start thread([])]: client.ConfiguredRMFailoverProxyProvider (ConfiguredRMFailoverProxyProvider.java:performFailover(100)) - Failing over to rm2