From 5baec0e619b063b736a2e38cad4fa0f91c362e5c Mon Sep 17 00:00:00 2001 From: Adam Antal Date: Mon, 13 Jan 2020 17:32:10 +0100 Subject: [PATCH] YARN-10083. Provide utility to ask whether an application is in final status (cherry picked from commit 3e5dee08f00546bb943ed8372453f05af34c470e) Conflicts: hadoop-tools/hadoop-dynamometer/hadoop-dynamometer-infra/src/main/java/org/apache/hadoop/tools/dynamometer/Client.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/cli/LogsCLI.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/logaggregation/AggregatedLogDeletionService.java --- .../apache/hadoop/mapred/NotRunningJob.java | 5 ++-- .../org/apache/hadoop/mapred/YARNRunner.java | 5 ++-- .../yarn/client/cli/ApplicationCLI.java | 9 ++---- .../hadoop/yarn/client/cli/LogsCLI.java | 21 ++++--------- .../AggregatedLogDeletionService.java | 5 +--- .../yarn/logaggregation/LogToolUtils.java | 30 +++++++++++++++++++ .../org/apache/hadoop/yarn/util/Apps.java | 14 +++++++++ ...licationHistoryManagerOnTimelineStore.java | 10 ++----- .../hadoop/yarn/server/webapp/AppBlock.java | 16 +++------- .../hadoop/yarn/server/webapp/AppsBlock.java | 7 ++--- .../hadoop/yarn/server/webapp/LogServlet.java | 5 ++-- .../server/webapp/LogWebServiceUtils.java | 6 ---- .../resourcemanager/webapp/RMAppsBlock.java | 7 ++--- .../timeline/EntityGroupFSTimelineStore.java | 10 ++----- 14 files changed, 74 insertions(+), 76 deletions(-) diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/main/java/org/apache/hadoop/mapred/NotRunningJob.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/main/java/org/apache/hadoop/mapred/NotRunningJob.java index 152c8af1941..89de9bd494a 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/main/java/org/apache/hadoop/mapred/NotRunningJob.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/main/java/org/apache/hadoop/mapred/NotRunningJob.java @@ -69,6 +69,7 @@ import org.apache.hadoop.yarn.conf.YarnConfiguration; import org.apache.hadoop.yarn.factories.RecordFactory; import org.apache.hadoop.yarn.factory.providers.RecordFactoryProvider; +import org.apache.hadoop.yarn.util.Apps; public class NotRunningJob implements MRClientProtocol { @@ -138,9 +139,7 @@ public GetJobReportResponse getJobReport(GetJobReportRequest request) jobReport.setUser(applicationReport.getUser()); jobReport.setStartTime(applicationReport.getStartTime()); YarnApplicationState state = applicationReport.getYarnApplicationState(); - if (state == YarnApplicationState.KILLED - || state == YarnApplicationState.FAILED - || state == YarnApplicationState.FINISHED) { + if (Apps.isApplicationFinalState(state)) { jobReport.setDiagnostics(applicationReport.getDiagnostics()); } jobReport.setJobName(applicationReport.getName()); diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/main/java/org/apache/hadoop/mapred/YARNRunner.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/main/java/org/apache/hadoop/mapred/YARNRunner.java index 7d33ed23d92..a9356e28a7d 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/main/java/org/apache/hadoop/mapred/YARNRunner.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/main/java/org/apache/hadoop/mapred/YARNRunner.java @@ -94,6 +94,7 @@ import org.apache.hadoop.yarn.factories.RecordFactory; import org.apache.hadoop.yarn.factory.providers.RecordFactoryProvider; import org.apache.hadoop.yarn.security.client.RMDelegationTokenSelector; +import org.apache.hadoop.yarn.util.Apps; import org.apache.hadoop.yarn.util.ConverterUtils; import org.apache.hadoop.yarn.util.UnitsConversionUtil; import org.apache.hadoop.yarn.util.resource.ResourceUtils; @@ -899,9 +900,7 @@ private void killUnFinishedApplication(ApplicationId appId) } catch (YarnException e) { throw new IOException(e); } - if (application.getYarnApplicationState() == YarnApplicationState.FINISHED - || application.getYarnApplicationState() == YarnApplicationState.FAILED - || application.getYarnApplicationState() == YarnApplicationState.KILLED) { + if (Apps.isApplicationFinalState(application.getYarnApplicationState())) { return; } killApplication(appId); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/cli/ApplicationCLI.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/cli/ApplicationCLI.java index 71f4c132091..2b7195ad7b4 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/cli/ApplicationCLI.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/cli/ApplicationCLI.java @@ -56,6 +56,7 @@ import org.apache.hadoop.yarn.exceptions.ApplicationNotFoundException; import org.apache.hadoop.yarn.exceptions.ContainerNotFoundException; import org.apache.hadoop.yarn.exceptions.YarnException; +import org.apache.hadoop.yarn.util.Apps; import org.apache.hadoop.yarn.util.Times; import com.google.common.annotations.VisibleForTesting; @@ -1058,9 +1059,7 @@ private void killApplication(String applicationId) throws YarnException, throw e; } - if (appReport.getYarnApplicationState() == YarnApplicationState.FINISHED - || appReport.getYarnApplicationState() == YarnApplicationState.KILLED - || appReport.getYarnApplicationState() == YarnApplicationState.FAILED) { + if (Apps.isApplicationFinalState(appReport.getYarnApplicationState())) { sysout.println("Application " + applicationId + " has already finished "); } else { sysout.println("Killing application " + applicationId); @@ -1075,9 +1074,7 @@ private void moveApplicationAcrossQueues(String applicationId, String queue) throws YarnException, IOException { ApplicationId appId = ApplicationId.fromString(applicationId); ApplicationReport appReport = client.getApplicationReport(appId); - if (appReport.getYarnApplicationState() == YarnApplicationState.FINISHED - || appReport.getYarnApplicationState() == YarnApplicationState.KILLED - || appReport.getYarnApplicationState() == YarnApplicationState.FAILED) { + if (Apps.isApplicationFinalState(appReport.getYarnApplicationState())) { sysout.println("Application " + applicationId + " has already finished "); } else { sysout.println("Moving application " + applicationId + " to queue " + queue); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/cli/LogsCLI.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/cli/LogsCLI.java index c6024429d35..4ada007f9cd 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/cli/LogsCLI.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/cli/LogsCLI.java @@ -85,6 +85,7 @@ import org.apache.hadoop.yarn.logaggregation.ContainerLogsRequest; import org.apache.hadoop.yarn.logaggregation.LogCLIHelpers; import org.apache.hadoop.yarn.logaggregation.LogToolUtils; +import org.apache.hadoop.yarn.util.Apps; import org.apache.hadoop.yarn.webapp.util.WebAppUtils; import org.apache.hadoop.yarn.webapp.util.YarnWebServiceUtils; import org.codehaus.jettison.json.JSONArray; @@ -354,8 +355,8 @@ private int runCommand(String[] args) throws Exception { ContainerLogsRequest request = new ContainerLogsRequest(appId, - isApplicationFinished(appState), appOwner, nodeAddress, null, - containerIdStr, localDir, logs, bytes, null); + Apps.isApplicationFinalState(appState), appOwner, nodeAddress, + null, containerIdStr, localDir, logs, bytes, null); if (showContainerLogInfo) { return showContainerLogInfo(request, logCliHelper); @@ -643,12 +644,6 @@ public ContainerReport getContainerReport(String containerIdStr) ContainerId.fromString(containerIdStr)); } - private boolean isApplicationFinished(YarnApplicationState appState) { - return appState == YarnApplicationState.FINISHED - || appState == YarnApplicationState.FAILED - || appState == YarnApplicationState.KILLED; - } - private int printAMContainerLogs(Configuration conf, ContainerLogsRequest request, List amContainers, LogCLIHelpers logCliHelper, boolean useRegex, boolean ignoreSizeLimit) @@ -1377,14 +1372,8 @@ private void outputContainerLogMeta(String containerId, String nodeId, @VisibleForTesting public ClientResponse getResponeFromNMWebService(Configuration conf, Client webServiceClient, ContainerLogsRequest request, String logFile) { - WebResource webResource = - webServiceClient.resource(WebAppUtils.getHttpSchemePrefix(conf) - + request.getNodeHttpAddress()); - return webResource.path("ws").path("v1").path("node") - .path("containers").path(request.getContainerId()).path("logs") - .path(logFile) - .queryParam("size", Long.toString(request.getBytes())) - .accept(MediaType.TEXT_PLAIN).get(ClientResponse.class); + return LogToolUtils.getResponeFromNMWebService( + conf, webServiceClient, request, logFile); } @VisibleForTesting diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/logaggregation/AggregatedLogDeletionService.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/logaggregation/AggregatedLogDeletionService.java index b251862f4c3..b29082339ab 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/logaggregation/AggregatedLogDeletionService.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/logaggregation/AggregatedLogDeletionService.java @@ -43,7 +43,6 @@ import org.apache.hadoop.yarn.exceptions.YarnException; import org.apache.hadoop.yarn.logaggregation.filecontroller.LogAggregationFileController; import org.apache.hadoop.yarn.logaggregation.filecontroller.LogAggregationFileControllerFactory; -import org.apache.hadoop.yarn.util.ConverterUtils; import com.google.common.annotations.VisibleForTesting; @@ -184,9 +183,7 @@ private static boolean isApplicationTerminated(ApplicationId appId, throw new IOException(e); } YarnApplicationState currentState = appReport.getYarnApplicationState(); - return currentState == YarnApplicationState.FAILED - || currentState == YarnApplicationState.KILLED - || currentState == YarnApplicationState.FINISHED; + return Apps.isApplicationFinalState(currentState); } public ApplicationClientProtocol getRMClient() { diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/logaggregation/LogToolUtils.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/logaggregation/LogToolUtils.java index a6cc159f85a..a0ecb8c0d42 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/logaggregation/LogToolUtils.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/logaggregation/LogToolUtils.java @@ -28,8 +28,16 @@ import java.nio.charset.Charset; import java.nio.file.Files; import java.nio.file.Paths; + +import com.sun.jersey.api.client.Client; +import com.sun.jersey.api.client.ClientResponse; +import com.sun.jersey.api.client.WebResource; import org.apache.commons.lang3.StringUtils; +import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.Path; +import org.apache.hadoop.yarn.webapp.util.WebAppUtils; + +import javax.ws.rs.core.MediaType; /** * This class contains several utility function which could be used in different @@ -184,4 +192,26 @@ public static PrintStream createPrintStream(String localDir, String nodeId, } return out; } + + /** + * Redirect the {@link ContainerLogsRequest} to the NodeManager's + * NMWebServices. + * + * @param conf Configuration object + * @param webServiceClient client + * @param request the request for container logs + * @param logFile name of the log file + * @return response from NMWebServices + */ + public static ClientResponse getResponeFromNMWebService(Configuration conf, + Client webServiceClient, ContainerLogsRequest request, String logFile) { + WebResource webResource = + webServiceClient.resource(WebAppUtils.getHttpSchemePrefix(conf) + + request.getNodeHttpAddress()); + return webResource.path("ws").path("v1").path("node") + .path("containers").path(request.getContainerId()).path("logs") + .path(logFile) + .queryParam("size", Long.toString(request.getBytes())) + .accept(MediaType.TEXT_PLAIN).get(ClientResponse.class); + } } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/Apps.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/Apps.java index 1ea7646e32e..a53ae039781 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/Apps.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/Apps.java @@ -40,6 +40,7 @@ import org.apache.hadoop.yarn.api.ApplicationConstants; import org.apache.hadoop.yarn.api.records.ApplicationId; import org.apache.hadoop.yarn.api.records.ContainerExitStatus; +import org.apache.hadoop.yarn.api.records.YarnApplicationState; import org.apache.hadoop.yarn.exceptions.YarnRuntimeException; /** @@ -301,4 +302,17 @@ public static boolean shouldCountTowardsNodeBlacklisting(int exitStatus) { return true; } } + + /** + * Returns whether a given application state is final: FINISHED, + * FAILED or KILLED. + * + * @param appState application state + * @return whether the appState is final. + */ + public static boolean isApplicationFinalState(YarnApplicationState appState) { + return appState == YarnApplicationState.FINISHED + || appState == YarnApplicationState.FAILED + || appState == YarnApplicationState.KILLED; + } } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/ApplicationHistoryManagerOnTimelineStore.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/ApplicationHistoryManagerOnTimelineStore.java index 3558c503302..288eae0302e 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/ApplicationHistoryManagerOnTimelineStore.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/ApplicationHistoryManagerOnTimelineStore.java @@ -64,7 +64,7 @@ import org.apache.hadoop.yarn.server.timeline.NameValuePair; import org.apache.hadoop.yarn.server.timeline.TimelineDataManager; import org.apache.hadoop.yarn.server.timeline.TimelineReader.Field; -import org.apache.hadoop.yarn.util.ConverterUtils; +import org.apache.hadoop.yarn.util.Apps; import org.apache.hadoop.yarn.webapp.util.WebAppUtils; import com.google.common.annotations.VisibleForTesting; @@ -408,7 +408,7 @@ private static ApplicationReportExt convertToApplicationReport( } if (eventInfo.containsKey( ApplicationMetricsConstants.STATE_EVENT_INFO)) { - if (!isFinalState(state)) { + if (!Apps.isApplicationFinalState(state)) { state = YarnApplicationState.valueOf(eventInfo.get( ApplicationMetricsConstants.STATE_EVENT_INFO).toString()); } @@ -470,12 +470,6 @@ private static long parseLong(Map entityInfo, return result; } - private static boolean isFinalState(YarnApplicationState state) { - return state == YarnApplicationState.FINISHED - || state == YarnApplicationState.FAILED - || state == YarnApplicationState.KILLED; - } - private static ApplicationAttemptReport convertToApplicationAttemptReport( TimelineEntity entity) { String host = null; diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/webapp/AppBlock.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/webapp/AppBlock.java index 0c7a5361712..17d9bf2d9a9 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/webapp/AppBlock.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/webapp/AppBlock.java @@ -174,7 +174,7 @@ public ApplicationReport run() throws Exception { && conf.getBoolean(YarnConfiguration.RM_WEBAPP_UI_ACTIONS_ENABLED, YarnConfiguration.DEFAULT_RM_WEBAPP_UI_ACTIONS_ENABLED) && !unsecuredUIForSecuredCluster - && !isAppInFinalState(app)) { + && !Apps.isApplicationFinalState(app.getAppState())) { // Application Kill html.div() .button() @@ -251,11 +251,9 @@ private void generateOverviewTable(AppInfo app, String schedulerPath, || app.getTrackingUrl().equals(UNAVAILABLE) ? null : root_url(app .getTrackingUrl()), app.getTrackingUrl() == null - || app.getTrackingUrl().equals(UNAVAILABLE) ? "Unassigned" : app - .getAppState() == YarnApplicationState.FINISHED - || app.getAppState() == YarnApplicationState.FAILED - || app.getAppState() == YarnApplicationState.KILLED ? "History" - : "ApplicationMaster"); + || app.getTrackingUrl().equals(UNAVAILABLE) ? "Unassigned" : + Apps.isApplicationFinalState(app.getAppState()) ? + "History" : "ApplicationMaster"); if (webUiType != null && webUiType.equals(YarnWebParams.RM_WEB_UI)) { LogAggregationStatus status = getLogAggregationStatus(); @@ -446,10 +444,4 @@ public static String getCSRFHeaderString(Configuration conf) { } return ret; } - - private boolean isAppInFinalState(AppInfo app) { - return app.getAppState() == YarnApplicationState.FINISHED - || app.getAppState() == YarnApplicationState.FAILED - || app.getAppState() == YarnApplicationState.KILLED; - } } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/webapp/AppsBlock.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/webapp/AppsBlock.java index 29843b53ab2..b89ca02e09d 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/webapp/AppsBlock.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/webapp/AppsBlock.java @@ -42,6 +42,7 @@ import org.apache.hadoop.yarn.api.records.YarnApplicationState; import org.apache.hadoop.yarn.exceptions.YarnException; import org.apache.hadoop.yarn.server.webapp.dao.AppInfo; +import org.apache.hadoop.yarn.util.Apps; import org.apache.hadoop.yarn.webapp.BadRequestException; import org.apache.hadoop.yarn.webapp.hamlet2.Hamlet; import org.apache.hadoop.yarn.webapp.hamlet2.Hamlet.TABLE; @@ -211,10 +212,8 @@ protected void renderData(Block html) { String trackingUI = app.getTrackingUrl() == null || app.getTrackingUrl().equals(UNAVAILABLE) - ? "Unassigned" - : app.getAppState() == YarnApplicationState.FINISHED - || app.getAppState() == YarnApplicationState.FAILED - || app.getAppState() == YarnApplicationState.KILLED + ? "Unassigned" : + Apps.isApplicationFinalState(app.getAppState()) ? "History" : "ApplicationMaster"; appsTableData.append(trackingURL == null ? "#" : "href='" + trackingURL) .append("'>").append(trackingUI).append("\"],\n"); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/webapp/LogServlet.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/webapp/LogServlet.java index 1bddf6dfda7..39e0ae308b5 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/webapp/LogServlet.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/webapp/LogServlet.java @@ -27,6 +27,7 @@ import org.apache.hadoop.yarn.api.records.ApplicationId; import org.apache.hadoop.yarn.api.records.ContainerId; import org.apache.hadoop.yarn.logaggregation.filecontroller.LogAggregationFileControllerFactory; +import org.apache.hadoop.yarn.util.Apps; import org.apache.hadoop.yarn.webapp.BadRequestException; import org.apache.hadoop.yarn.webapp.NotFoundException; import org.codehaus.jettison.json.JSONException; @@ -97,7 +98,7 @@ public Response getContainerLogsInfo(HttpServletRequest req, } // if the application finishes, directly find logs // from HDFS. - if (LogWebServiceUtils.isFinishedState(appInfo.getAppState())) { + if (Apps.isApplicationFinalState(appInfo.getAppState())) { return LogWebServiceUtils .getContainerLogMeta(factory, appId, null, null, containerIdStr, false); @@ -197,7 +198,7 @@ public Response getLogFile(HttpServletRequest req, String containerIdStr, filename, format, length, false); } String appOwner = appInfo.getUser(); - if (LogWebServiceUtils.isFinishedState(appInfo.getAppState())) { + if (Apps.isApplicationFinalState(appInfo.getAppState())) { // directly find logs from HDFS. return LogWebServiceUtils .sendStreamOutputResponse(factory, appId, appOwner, null, diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/webapp/LogWebServiceUtils.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/webapp/LogWebServiceUtils.java index bc301bbb2d8..defde4d39aa 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/webapp/LogWebServiceUtils.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/webapp/LogWebServiceUtils.java @@ -222,12 +222,6 @@ public static boolean isRunningState(YarnApplicationState appState) { return appState == YarnApplicationState.RUNNING; } - public static boolean isFinishedState(YarnApplicationState appState) { - return appState == YarnApplicationState.FINISHED - || appState == YarnApplicationState.FAILED - || appState == YarnApplicationState.KILLED; - } - protected static UserGroupInformation getUser(HttpServletRequest req) { String remoteUser = req.getRemoteUser(); UserGroupInformation callerUGI = null; diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMAppsBlock.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMAppsBlock.java index b1c0cd94ed4..671880c5551 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMAppsBlock.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMAppsBlock.java @@ -38,6 +38,7 @@ import org.apache.hadoop.yarn.server.resourcemanager.rmapp.attempt.RMAppAttempt; import org.apache.hadoop.yarn.server.webapp.AppsBlock; import org.apache.hadoop.yarn.server.webapp.dao.AppInfo; +import org.apache.hadoop.yarn.util.Apps; import org.apache.hadoop.yarn.webapp.View; import org.apache.hadoop.yarn.webapp.hamlet2.Hamlet; import org.apache.hadoop.yarn.webapp.hamlet2.Hamlet.TABLE; @@ -178,10 +179,8 @@ protected void renderData(Block html) { app.getTrackingUrl() == null || app.getTrackingUrl().equals(UNAVAILABLE) || app.getAppState() == YarnApplicationState.NEW ? "Unassigned" - : app.getAppState() == YarnApplicationState.FINISHED - || app.getAppState() == YarnApplicationState.FAILED - || app.getAppState() == YarnApplicationState.KILLED ? "History" - : "ApplicationMaster"; + : Apps.isApplicationFinalState(app.getAppState()) ? + "History" : "ApplicationMaster"; appsTableData.append(trackingURL == null ? "#" : "href='" + trackingURL) .append("'>").append(trackingUI).append("\",").append("\"") .append(blacklistedNodesCount).append("\"],\n"); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timeline-pluginstorage/src/main/java/org/apache/hadoop/yarn/server/timeline/EntityGroupFSTimelineStore.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timeline-pluginstorage/src/main/java/org/apache/hadoop/yarn/server/timeline/EntityGroupFSTimelineStore.java index 80baf897376..f8f0aed44c8 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timeline-pluginstorage/src/main/java/org/apache/hadoop/yarn/server/timeline/EntityGroupFSTimelineStore.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timeline-pluginstorage/src/main/java/org/apache/hadoop/yarn/server/timeline/EntityGroupFSTimelineStore.java @@ -40,7 +40,6 @@ import org.apache.hadoop.yarn.api.records.ApplicationAttemptId; import org.apache.hadoop.yarn.api.records.ApplicationId; import org.apache.hadoop.yarn.api.records.ApplicationReport; -import org.apache.hadoop.yarn.api.records.YarnApplicationState; import org.apache.hadoop.yarn.api.records.timeline.TimelineDomain; import org.apache.hadoop.yarn.api.records.timeline.TimelineDomains; import org.apache.hadoop.yarn.api.records.timeline.TimelineEntities; @@ -54,6 +53,7 @@ import org.apache.hadoop.yarn.exceptions.YarnException; import org.apache.hadoop.yarn.server.timeline.TimelineDataManager.CheckAcl; import org.apache.hadoop.yarn.server.timeline.security.TimelineACLsManager; +import org.apache.hadoop.yarn.util.Apps; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -102,11 +102,6 @@ private static final FsPermission DONE_DIR_PERMISSION = new FsPermission((short) 0700); - private static final EnumSet - APP_FINAL_STATES = EnumSet.of( - YarnApplicationState.FAILED, - YarnApplicationState.KILLED, - YarnApplicationState.FINISHED); // Active dir: /appId/attemptId/cacheId.log // Done dir: /cluster_ts/hash1/hash2/appId/attemptId/cacheId.log private static final String APP_DONE_DIR_PREFIX_FORMAT = @@ -615,8 +610,7 @@ private static synchronized AppState getAppState(ApplicationId appId, AppState appState = AppState.ACTIVE; try { ApplicationReport report = yarnClient.getApplicationReport(appId); - YarnApplicationState yarnState = report.getYarnApplicationState(); - if (APP_FINAL_STATES.contains(yarnState)) { + if (Apps.isApplicationFinalState(report.getYarnApplicationState())) { appState = AppState.COMPLETED; } } catch (ApplicationNotFoundException e) { -- 2.21.0