diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/cli/LogsCLI.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/cli/LogsCLI.java index 9a520a1..a66b3c7 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/cli/LogsCLI.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/cli/LogsCLI.java @@ -47,6 +47,7 @@ import org.apache.hadoop.conf.Configured; import org.apache.hadoop.security.UserGroupInformation; import org.apache.hadoop.util.Tool; +import org.apache.hadoop.yarn.api.records.ApplicationAttemptId; import org.apache.hadoop.yarn.api.records.ApplicationAttemptReport; import org.apache.hadoop.yarn.api.records.ApplicationId; import org.apache.hadoop.yarn.api.records.ApplicationReport; @@ -58,8 +59,8 @@ import org.apache.hadoop.yarn.exceptions.YarnException; import org.apache.hadoop.yarn.logaggregation.LogCLIHelpers; import org.apache.hadoop.yarn.logaggregation.ContainerLogsRequest; -import org.apache.hadoop.yarn.util.ConverterUtils; import org.apache.hadoop.yarn.util.Times; +import org.apache.hadoop.yarn.webapp.NotFoundException; import org.apache.hadoop.yarn.webapp.util.WebAppUtils; import org.codehaus.jettison.json.JSONArray; import org.codehaus.jettison.json.JSONException; @@ -470,19 +471,6 @@ private int printContainerLogsForFinishedApplicationWithoutNodeId( newOptions); } - @Private - @VisibleForTesting - public ContainerReport getContainerReport(String containerIdStr) - throws YarnException, IOException { - YarnClient yarnClient = createYarnClient(); - try { - return yarnClient.getContainerReport( - ContainerId.fromString(containerIdStr)); - } finally { - yarnClient.close(); - } - } - private boolean isApplicationFinished(YarnApplicationState appState) { return appState == YarnApplicationState.FINISHED || appState == YarnApplicationState.FAILED @@ -498,6 +486,9 @@ private int printAMContainerLogs(Configuration conf, boolean getAMContainerLists = false; String appId = request.getAppId().toString(); String errorMessage = ""; + // We will call RM webservice to get all AppAttempts information. + // If we get nothing, we will try to call AHS webservice to get AppAttempts + // which includes nodeAddress for the AM Containers. try { amContainersList = getAMContainerInfoForRMWebService(conf, appId); if (amContainersList != null && !amContainersList.isEmpty()) { @@ -535,6 +526,9 @@ private int printAMContainerLogs(Configuration conf, System.err.println("Unable to get AM container informations " + "for the application:" + appId); System.err.println(errorMessage); + System.err.println("Can not get AMContainers logs for " + + "the application:" + appId + " with the appOwner:" + + request.getAppOwner()); return -1; } @@ -575,19 +569,12 @@ private void outputAMContainerLogs(ContainerLogsRequest request, if (request.isAppFinished()) { if (containerId != null && !containerId.isEmpty()) { - if (nodeId == null || nodeId.isEmpty()) { - try { - nodeId = - getContainerReport(containerId).getAssignedNode().toString(); - request.setNodeId(nodeId); - } catch (Exception ex) { - System.err.println(ex); - nodeId = null; - } - } if (nodeId != null && !nodeId.isEmpty()) { printContainerLogsForFinishedApplication(request, logCliHelper); + } else { + printContainerLogsForFinishedApplicationWithoutNodeId(request, + logCliHelper); } } } else { @@ -740,38 +727,8 @@ private int fetchAMContainerLogs(ContainerLogsRequest request, logFiles = Arrays.asList("syslog"); } request.setLogTypes(logFiles); - // If the application is running, we will call the RM WebService - // to get the AppAttempts which includes the nodeHttpAddress - // and containerId for all the AM Containers. - // After that, we will call NodeManager webService to get the - // related logs - if (!request.isAppFinished()) { - return printAMContainerLogs(getConf(), request, amContainersList, - logCliHelper); - } else { - // If the application is in the final state, we will call RM webservice - // to get all AppAttempts information first. If we get nothing, - // we will try to call AHS webservice to get related AppAttempts - // which includes nodeAddress for the AM Containers. - // After that, we will use nodeAddress and containerId - // to get logs from HDFS directly. - if (getConf().getBoolean(YarnConfiguration.APPLICATION_HISTORY_ENABLED, - YarnConfiguration.DEFAULT_APPLICATION_HISTORY_ENABLED)) { - return printAMContainerLogs(getConf(), request, amContainersList, - logCliHelper); - } else { - ApplicationId appId = request.getAppId(); - String appOwner = request.getAppOwner(); - System.err.println("Can not get AMContainers logs for " - + "the application:" + appId + " with the appOwner:" + appOwner); - System.err.println("This application:" + appId + " has finished." - + " Please enable the application-history service or explicitly" - + " use 'yarn logs -applicationId " - + "-containerId --nodeAddress ' " - + "to get the container logs."); - return -1; - } - } + return printAMContainerLogs(getConf(), request, amContainersList, + logCliHelper); } private int fetchContainerLogs(ContainerLogsRequest request, @@ -783,61 +740,56 @@ private int fetchContainerLogs(ContainerLogsRequest request, String appOwner = request.getAppOwner(); boolean isAppFinished = request.isAppFinished(); List logFiles = request.getLogTypes(); - // if we provide the node address and the application is in the final - // state, we could directly get logs from HDFS. - if (nodeAddress != null && isAppFinished) { + // if application is in the final state, + // we could directly get logs from HDFS. + if (isAppFinished) { // if user specified "ALL" as the logFiles param, pass empty list // to logCliHelper so that it fetches all the logs - return printContainerLogsForFinishedApplication( - request, logCliHelper); + if (nodeAddress != null) { + return printContainerLogsForFinishedApplication( + request, logCliHelper); + } else { + return printContainerLogsForFinishedApplicationWithoutNodeId( + request, logCliHelper); + } } + // If the application is not in the final state, + // we will get the NodeHttpAddress, then get the container logs + // by calling NodeManager webservice. String nodeHttpAddress = null; String nodeId = null; try { // If the nodeAddress is not provided, we will try to get // the ContainerReport. In the containerReport, we could get // nodeAddress and nodeHttpAddress - ContainerReport report = getContainerReport(containerIdStr); - nodeHttpAddress = - report.getNodeHttpAddress().replaceFirst( - WebAppUtils.getHttpSchemePrefix(getConf()), ""); - nodeId = report.getAssignedNode().toString(); + ContainerId containerId = ContainerId.fromString(containerIdStr); + JSONObject containerInfo = getContainerInfo(getConf(), containerId); + nodeHttpAddress = containerInfo.getString("nodeHttpAddress") + .replaceFirst(WebAppUtils.getHttpSchemePrefix(getConf()), ""); + nodeId = containerInfo.getString("nodeId"); request.setNodeId(nodeId); request.setNodeHttpAddress(nodeHttpAddress); - } catch (IOException | YarnException ex) { - if (isAppFinished) { - return printContainerLogsForFinishedApplicationWithoutNodeId( - request, logCliHelper); - } else { - System.err.println("Unable to get logs for this container:" - + containerIdStr + "for the application:" + appIdStr - + " with the appOwner: " + appOwner); - System.err.println("The application: " + appIdStr - + " is still running, and we can not get Container report " - + "for the container: " + containerIdStr +". Please try later " - + "or after the application finishes."); - return -1; - } + } catch (ClientHandlerException | UniformInterfaceException + | JSONException ex) { + System.err.println(ex.getMessage()); + System.err.println("Unable to get logs for this container:" + + containerIdStr + "for the application:" + appIdStr + + " with the appOwner: " + appOwner); + System.err.println("The application: " + appIdStr + + " is still running, and we can not get Container report " + + "for the container: " + containerIdStr +". Please try later " + + "or after the application finishes."); + return -1; } - // If the application is not in the final state, - // we will provide the NodeHttpAddress and get the container logs - // by calling NodeManager webservice. - if (!isAppFinished) { - // if we do not specify the value for CONTAINER_LOG_FILES option, - // we will only output syslog - if (logFiles == null || logFiles.isEmpty()) { - logFiles = Arrays.asList("syslog"); - } - request.setLogTypes(logFiles); - resultCode = printContainerLogsFromRunningApplication(getConf(), request, - logCliHelper); - } else { - // If the application is in the final state, we will directly - // get the container logs from HDFS. - resultCode = printContainerLogsForFinishedApplication( - request, logCliHelper); + + // if we do not specify the value for CONTAINER_LOG_FILES option, + // we will only output syslog + if (logFiles == null || logFiles.isEmpty()) { + logFiles = Arrays.asList("syslog"); } - return resultCode; + request.setLogTypes(logFiles); + return printContainerLogsFromRunningApplication(getConf(), request, + logCliHelper); } private int fetchApplicationLogs(ContainerLogsRequest options, @@ -967,4 +919,43 @@ private boolean isFileMatching(String fileType, yarnClient.close(); } } + + @Private + @VisibleForTesting + public JSONObject getContainerInfo(Configuration conf, + ContainerId containerId) throws ClientHandlerException, + UniformInterfaceException, JSONException { + String webAppAddress = WebAppUtils.getHttpSchemePrefix(conf) + + WebAppUtils.getWebAppBindURL(conf, YarnConfiguration.RM_BIND_HOST, + WebAppUtils.getRMWebAppURLWithoutScheme(conf)); + JSONObject object = null; + try { + object = getContainerInfoFromWebService(conf, + webAppAddress, containerId); + } catch(NotFoundException ex) { + webAppAddress = + WebAppUtils.getHttpSchemePrefix(conf) + + WebAppUtils.getAHSWebAppURLWithoutScheme(conf); + object = getContainerInfoFromWebService(conf, webAppAddress, + containerId); + } + return object; + } + + private JSONObject getContainerInfoFromWebService(Configuration conf, + String serviceWebAddress, ContainerId containerId) throws + ClientHandlerException, UniformInterfaceException, JSONException { + ApplicationAttemptId attemptId = containerId.getApplicationAttemptId(); + ApplicationId appId = attemptId.getApplicationId(); + Client webServiceClient = Client.create(); + WebResource webResource = webServiceClient.resource(serviceWebAddress); + + ClientResponse response = + webResource.path("ws").path("v1").path("cluster").path("apps") + .path(appId.toString()).path("appattempts") + .path(attemptId.toString()).path("containers") + .path(containerId.toString()).accept(MediaType.APPLICATION_JSON) + .get(ClientResponse.class); + return response.getEntity(JSONObject.class); + } } diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/cli/TestLogsCLI.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/cli/TestLogsCLI.java index 85287ea..a309a69 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/cli/TestLogsCLI.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/cli/TestLogsCLI.java @@ -29,6 +29,8 @@ import static org.mockito.Mockito.times; import static org.mockito.Mockito.verify; +import com.sun.jersey.api.client.ClientHandlerException; +import com.sun.jersey.api.client.UniformInterfaceException; import java.io.BufferedReader; import java.io.ByteArrayOutputStream; import java.io.DataOutputStream; @@ -70,6 +72,7 @@ import org.apache.hadoop.yarn.logaggregation.ContainerLogsRequest; import org.apache.hadoop.yarn.logaggregation.LogAggregationUtils; import org.apache.hadoop.yarn.logaggregation.LogCLIHelpers; +import org.codehaus.jettison.json.JSONException; import org.codehaus.jettison.json.JSONObject; import org.junit.Assert; import org.junit.Before; @@ -309,12 +312,13 @@ public void testFetchApplictionLogs() throws Exception { YarnApplicationState.FINISHED, ugi.getShortUserName()); LogsCLI cli = new LogsCLIForTest(mockYarnClient) { @Override - public ContainerReport getContainerReport(String containerIdStr) - throws YarnException, IOException { - ContainerReport mockReport = mock(ContainerReport.class); - doReturn(nodeId).when(mockReport).getAssignedNode(); - doReturn("http://localhost:2345").when(mockReport).getNodeHttpAddress(); - return mockReport; + public JSONObject getContainerInfo(Configuration conf, + ContainerId containerId) throws ClientHandlerException, + UniformInterfaceException, JSONException { + JSONObject obj = new JSONObject(); + obj.put("nodeId", nodeId); + obj.put("nodeHttpAddress", "http://localhost:2345"); + return obj; } }; cli.setConf(configuration);