diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/impl/YarnClientImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/impl/YarnClientImpl.java index 5eda2c8..622b0aa 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/impl/YarnClientImpl.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/impl/YarnClientImpl.java @@ -332,7 +332,7 @@ private void addTimelineDelegationToken( } credentials.addToken(timelineService, timelineDelegationToken); if (LOG.isDebugEnabled()) { - LOG.debug("Add timline delegation token into credentials: " + LOG.debug("Add timeline delegation token to credentials: " + timelineDelegationToken); } DataOutputBuffer dob = new DataOutputBuffer(); @@ -348,8 +348,10 @@ private void addTimelineDelegationToken( return timelineClient.getDelegationToken(timelineDTRenewer); } catch (Exception e ) { if (timelineServiceBestEffort) { - LOG.warn("Failed to get delegation token from the timeline server: " + LOG.warn("Failed to get delegation token from the timeline server;" + + " timeline client no longer publishing data: " + e.getMessage()); + LOG.debug("Full exception details", e); return null; } throw e; diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/client/api/impl/TimelineClientImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/client/api/impl/TimelineClientImpl.java index df6c7a4..1264bd5 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/client/api/impl/TimelineClientImpl.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/client/api/impl/TimelineClientImpl.java @@ -20,6 +20,7 @@ import java.io.File; import java.io.IOException; +import java.io.InterruptedIOException; import java.lang.reflect.UndeclaredThrowableException; import java.net.ConnectException; import java.net.HttpURLConnection; @@ -177,11 +178,13 @@ public Object retryOn(TimelineClientRetryOp op) retried = false; // keep trying + Exception lastException; while (true) { try { // try perform the op, if fail, keep retrying return op.run(); } catch (IOException | RuntimeException e) { + lastException = e; // break if there's no retries left if (leftRetries == 0) { break; @@ -201,23 +204,35 @@ public Object retryOn(TimelineClientRetryOp op) Thread.sleep(retryInterval); } catch (InterruptedException ie) { LOG.warn("Client retry sleep interrupted! "); + throw (InterruptedIOException) + (new InterruptedIOException(ie.toString()).initCause(ie)); } } - throw new RuntimeException("Failed to connect to timeline server. " - + "Connection retries limit exceeded. " - + "The posted timeline event may be missing"); - }; + // reached only if the retry count has been exceeded. + // therefore, lastException no-null + LOG.warn("Failed to connect to timeline server. " + + "Connection retries limit (" + maxRetries + ") exceeded. " + + "The posted timeline event may be missing", lastException); + + if (lastException instanceof IOException) { + throw (IOException) lastException; + } else { + throw (RuntimeException) lastException; + } + } private void logException(Exception e, int leftRetries) { if (leftRetries > 0) { LOG.info("Exception caught by TimelineClientConnectionRetry," + " will try " + leftRetries + " more time(s).\nMessage: " + e.getMessage()); + LOG.debug("Failure", e); } else { // note that maxRetries may be -1 at the very beginning LOG.info("ConnectionException caught by TimelineClientConnectionRetry," + " will keep retrying.\nMessage: " + e.getMessage()); + LOG.debug("Failure", e); } } } @@ -244,8 +259,8 @@ public boolean shouldRetryOn(Exception e) { try { return (ClientResponse) connectionRetry.retryOn(jerseyRetryOp); } catch (IOException e) { - throw new ClientHandlerException("Jersey retry failed!\nMessage: " - + e.getMessage()); + throw new ClientHandlerException("Jersey retry failed against " + resURI + + "\nException: " + e, e); } } } @@ -324,20 +339,30 @@ public ClientResponse run() throws Exception { } }); } catch (UndeclaredThrowableException e) { - throw new IOException(e.getCause()); + Throwable cause = e.getCause(); + if (cause instanceof IOException) { + throw (IOException) cause; + } else { + throw new IOException(cause); + } } catch (InterruptedException ie) { - throw new IOException(ie); + throw (InterruptedIOException) + (new InterruptedIOException(ie.toString()).initCause(ie)); } if (resp == null || resp.getClientResponseStatus() != ClientResponse.Status.OK) { String msg = - "Failed to get the response from the timeline server."; - LOG.error(msg); - if (LOG.isDebugEnabled() && resp != null) { - String output = resp.getEntity(String.class); - LOG.debug("HTTP error code: " + resp.getStatus() - + " Server response : \n" + output); + "Failed to get the response from the timeline server " + resURI; + if (resp != null) { + int status = resp.getStatus(); + msg += " -status code=" + status; + if (LOG.isDebugEnabled()) { + String output = resp.getEntity(String.class); + LOG.debug("HTTP error code: " + status + + " Server response : \n" + output); + } } + LOG.error(msg); throw new YarnException(msg); } return resp; @@ -440,7 +465,8 @@ public Object run() throws IOException { } catch (UndeclaredThrowableException e) { throw new IOException(e.getCause()); } catch (InterruptedException e) { - throw new IOException(e); + throw (InterruptedIOException) + (new InterruptedIOException(e.toString()).initCause(e)); } }