From f2b3add82b07e90ac37eea8772f84666f5c8d3a3 Mon Sep 17 00:00:00 2001 From: Abdullah Yousufi Date: Fri, 1 Sep 2017 01:21:50 -0700 Subject: [PATCH] submit time Change-Id: I02fa3789c8018d83e2b4451c0ec3191edea4586b --- .../org/apache/hadoop/mapred/NotRunningJob.java | 2 +- .../hadoop/mapred/TestClientServiceDelegate.java | 4 +- .../org/apache/hadoop/mapred/TestYARNRunner.java | 2 +- .../hadoop/yarn/api/records/ApplicationReport.java | 27 +++++++++++-- .../proto/server/application_history_server.proto | 2 + .../src/main/proto/yarn_protos.proto | 1 + .../hadoop/yarn/client/ProtocolHATestBase.java | 2 +- .../hadoop/yarn/client/api/impl/TestAHSClient.java | 8 ++-- .../yarn/client/api/impl/TestYarnClient.java | 8 ++-- .../apache/hadoop/yarn/client/cli/TestYarnCLI.java | 44 +++++++++++----------- .../records/impl/pb/ApplicationReportPBImpl.java | 12 ++++++ .../hadoop/yarn/api/TestApplicatonReport.java | 2 +- .../ApplicationHistoryManagerImpl.java | 6 +-- .../ApplicationHistoryManagerOnTimelineStore.java | 18 +++++---- .../FileSystemApplicationHistoryStore.java | 5 ++- .../MemoryApplicationHistoryStore.java | 4 +- .../records/ApplicationFinishData.java | 11 +++++- .../records/ApplicationHistoryData.java | 20 +++++++++- .../impl/pb/ApplicationFinishDataPBImpl.java | 12 ++++++ .../ApplicationHistoryStoreTestUtils.java | 2 +- .../hadoop/yarn/server/utils/BuilderUtils.java | 3 +- .../apache/hadoop/yarn/server/webapp/AppBlock.java | 8 ++-- .../hadoop/yarn/server/webapp/AppsBlock.java | 5 ++- .../hadoop/yarn/server/webapp/WebPageUtils.java | 2 +- .../hadoop/yarn/server/webapp/dao/AppInfo.java | 14 ++++++- .../ahs/RMApplicationHistoryWriter.java | 4 +- .../yarn/server/resourcemanager/rmapp/RMApp.java | 10 ++++- .../server/resourcemanager/rmapp/RMAppImpl.java | 27 +++++++++++-- .../rmapp/attempt/RMAppAttemptImpl.java | 4 ++ .../webapp/FairSchedulerAppsBlock.java | 2 + .../server/resourcemanager/webapp/RMAppsBlock.java | 2 + .../server/resourcemanager/webapp/dao/AppInfo.java | 17 ++++++++- .../applicationsmanager/MockAsm.java | 19 ++++++++-- .../server/resourcemanager/rmapp/MockRMApp.java | 6 +++ .../resourcemanager/webapp/TestRMWebApp.java | 2 +- .../webapp/TestRMWebServicesApps.java | 15 ++++++-- .../src/site/markdown/ResourceManagerRest.md | 12 +++++- .../webapp/app/controllers/app-table-columns.js | 12 ++++++ .../src/main/webapp/app/models/yarn-app.js | 21 +++++++++-- .../src/main/webapp/app/serializers/yarn-app.js | 2 + .../webapp/app/templates/components/app-table.hbs | 6 +++ .../main/webapp/app/templates/yarn-app/info.hbs | 4 ++ 42 files changed, 301 insertions(+), 88 deletions(-) diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/main/java/org/apache/hadoop/mapred/NotRunningJob.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/main/java/org/apache/hadoop/mapred/NotRunningJob.java index 90635a68854..bfdc6db8abd 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/main/java/org/apache/hadoop/mapred/NotRunningJob.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/main/java/org/apache/hadoop/mapred/NotRunningJob.java @@ -89,7 +89,7 @@ private ApplicationReport getUnknownApplicationReport() { // used for a non running job return ApplicationReport.newInstance(unknownAppId, unknownAttemptId, "N/A", "N/A", "N/A", "N/A", 0, null, YarnApplicationState.NEW, "N/A", - "N/A", 0, 0, FinalApplicationStatus.UNDEFINED, null, "N/A", 0.0f, + "N/A", 0, 0, 0, FinalApplicationStatus.UNDEFINED, null, "N/A", 0.0f, YarnConfiguration.DEFAULT_APPLICATION_TYPE, null); } diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestClientServiceDelegate.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestClientServiceDelegate.java index b85f18db4eb..275456df837 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestClientServiceDelegate.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestClientServiceDelegate.java @@ -515,7 +515,7 @@ private ApplicationReport getFinishedApplicationReport() { appId, 0); return ApplicationReport.newInstance(appId, attemptId, "user", "queue", "appname", "host", 124, null, YarnApplicationState.FINISHED, - "diagnostics", "url", 0, 0, FinalApplicationStatus.SUCCEEDED, null, + "diagnostics", "url", 0, 0, 0, FinalApplicationStatus.SUCCEEDED, null, "N/A", 0.0f, YarnConfiguration.DEFAULT_APPLICATION_TYPE, null); } @@ -525,7 +525,7 @@ private ApplicationReport getRunningApplicationReport(String host, int port) { appId, 0); return ApplicationReport.newInstance(appId, attemptId, "user", "queue", "appname", host, port, null, YarnApplicationState.RUNNING, "diagnostics", - "url", 0, 0, FinalApplicationStatus.UNDEFINED, null, "N/A", 0.0f, + "url", 0, 0, 0, FinalApplicationStatus.UNDEFINED, null, "N/A", 0.0f, YarnConfiguration.DEFAULT_APPLICATION_TYPE, null); } diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestYARNRunner.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestYARNRunner.java index 55ddea6da38..7a636b7b63d 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestYARNRunner.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestYARNRunner.java @@ -206,7 +206,7 @@ public ClientServiceDelegate answer(InvocationOnMock invocation) .thenReturn( ApplicationReport.newInstance(appId, null, "tmp", "tmp", "tmp", "tmp", 0, null, YarnApplicationState.FINISHED, "tmp", "tmp", - 0l, 0l, FinalApplicationStatus.SUCCEEDED, null, null, 0f, + 0L, 0L, 0L, FinalApplicationStatus.SUCCEEDED, null, null, 0f, "tmp", null)); yarnRunner.killJob(jobId); verify(clientDelegate).killJob(jobId); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ApplicationReport.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ApplicationReport.java index 9e9ec3cd2c9..778246e3b3b 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ApplicationReport.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ApplicationReport.java @@ -58,7 +58,8 @@ public static ApplicationReport newInstance(ApplicationId applicationId, ApplicationAttemptId applicationAttemptId, String user, String queue, String name, String host, int rpcPort, Token clientToAMToken, YarnApplicationState state, String diagnostics, String url, - long startTime, long finishTime, FinalApplicationStatus finalStatus, + long startTime, long launchTime, long finishTime, + FinalApplicationStatus finalStatus, ApplicationResourceUsageReport appResources, String origTrackingUrl, float progress, String applicationType, Token amRmToken) { ApplicationReport report = Records.newRecord(ApplicationReport.class); @@ -74,6 +75,7 @@ public static ApplicationReport newInstance(ApplicationId applicationId, report.setDiagnostics(diagnostics); report.setTrackingUrl(url); report.setStartTime(startTime); + report.setLaunchTime(launchTime); report.setFinishTime(finishTime); report.setFinalApplicationStatus(finalStatus); report.setApplicationResourceUsageReport(appResources); @@ -90,7 +92,8 @@ public static ApplicationReport newInstance(ApplicationId applicationId, ApplicationAttemptId applicationAttemptId, String user, String queue, String name, String host, int rpcPort, Token clientToAMToken, YarnApplicationState state, String diagnostics, String url, - long startTime, long finishTime, FinalApplicationStatus finalStatus, + long startTime, long launchTime, long finishTime, + FinalApplicationStatus finalStatus, ApplicationResourceUsageReport appResources, String origTrackingUrl, float progress, String applicationType, Token amRmToken, Set tags, boolean unmanagedApplication, Priority priority, @@ -98,8 +101,8 @@ public static ApplicationReport newInstance(ApplicationId applicationId, ApplicationReport report = newInstance(applicationId, applicationAttemptId, user, queue, name, host, rpcPort, clientToAMToken, state, diagnostics, url, startTime, - finishTime, finalStatus, appResources, origTrackingUrl, progress, - applicationType, amRmToken); + launchTime, finishTime, finalStatus, appResources, origTrackingUrl, + progress, applicationType, amRmToken); report.setApplicationTags(tags); report.setUnmanagedApp(unmanagedApplication); report.setPriority(priority); @@ -283,6 +286,22 @@ public static ApplicationReport newInstance(ApplicationId applicationId, public abstract void setStartTime(long startTime); /** + * Get the launch time of the application. + * As startTime is essentially submitTime, this new field has been created + * to provide more information regarding the app's execution and + * startTime has been left untouched as changing it may break backwards + * compatibility with the v1 webservices endpoint. + * @return launch time of the application + */ + @Public + @Stable + public abstract long getLaunchTime(); + + @Private + @Unstable + public abstract void setLaunchTime(long launchTime); + + /** * Get the finish time of the application. * @return finish time of the application */ diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/server/application_history_server.proto b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/server/application_history_server.proto index 0fcf2ac61af..444fe291946 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/server/application_history_server.proto +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/server/application_history_server.proto @@ -36,6 +36,7 @@ message ApplicationHistoryDataProto { optional string diagnostics_info = 9; optional FinalApplicationStatusProto final_application_status = 10; optional YarnApplicationStateProto yarn_application_state = 11; + optional int64 launch_time = 12; } message ApplicationStartDataProto { @@ -54,6 +55,7 @@ message ApplicationFinishDataProto { optional string diagnostics_info = 3; optional FinalApplicationStatusProto final_application_status = 4; optional YarnApplicationStateProto yarn_application_state = 5; + optional int64 launch_time = 6; } message ApplicationAttemptHistoryDataProto { diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/yarn_protos.proto b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/yarn_protos.proto index 81ebd798bb9..4bfdfc73d66 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/yarn_protos.proto +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/yarn_protos.proto @@ -215,6 +215,7 @@ message ApplicationReportProto { optional string appNodeLabelExpression = 24; optional string amNodeLabelExpression = 25; repeated AppTimeoutsMapProto appTimeouts = 26; + optional int64 launchTime = 27; } message AppTimeoutsMapProto { diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/ProtocolHATestBase.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/ProtocolHATestBase.java index a8e91323a38..ee358c24609 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/ProtocolHATestBase.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/ProtocolHATestBase.java @@ -625,7 +625,7 @@ public ApplicationReport createFakeAppReport() { ApplicationReport.newInstance(appId, attemptId, "fakeUser", "fakeQueue", "fakeApplicationName", "localhost", 0, null, YarnApplicationState.FINISHED, "fake an application report", "", - 1000l, 1200l, FinalApplicationStatus.FAILED, null, "", 50f, + 1000L, 1000L, 1200L, FinalApplicationStatus.FAILED, null, "", 50f, "fakeApplicationType", null); return report; } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/api/impl/TestAHSClient.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/api/impl/TestAHSClient.java index c3e3c414771..f0e3ca2a4cc 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/api/impl/TestAHSClient.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/api/impl/TestAHSClient.java @@ -332,7 +332,7 @@ private void createAppReports() { ApplicationReport.newInstance(applicationId, ApplicationAttemptId.newInstance(applicationId, 1), "user", "queue", "appname", "host", 124, null, - YarnApplicationState.RUNNING, "diagnostics", "url", 0, 0, + YarnApplicationState.RUNNING, "diagnostics", "url", 0, 0, 0, FinalApplicationStatus.SUCCEEDED, null, "N/A", 0.53789f, "YARN", null); List applicationReports = @@ -389,7 +389,7 @@ private void createAppReports() { ApplicationReport.newInstance(applicationId2, ApplicationAttemptId.newInstance(applicationId2, 2), "user2", "queue2", "appname2", "host2", 125, null, - YarnApplicationState.FINISHED, "diagnostics2", "url2", 2, 2, + YarnApplicationState.FINISHED, "diagnostics2", "url2", 2, 2, 2, FinalApplicationStatus.SUCCEEDED, null, "N/A", 0.63789f, "NON-YARN", null); applicationReports.add(newApplicationReport2); @@ -399,7 +399,7 @@ private void createAppReports() { ApplicationReport.newInstance(applicationId3, ApplicationAttemptId.newInstance(applicationId3, 3), "user3", "queue3", "appname3", "host3", 126, null, - YarnApplicationState.RUNNING, "diagnostics3", "url3", 3, 3, + YarnApplicationState.RUNNING, "diagnostics3", "url3", 3, 3, 3, FinalApplicationStatus.SUCCEEDED, null, "N/A", 0.73789f, "MAPREDUCE", null); applicationReports.add(newApplicationReport3); @@ -409,7 +409,7 @@ private void createAppReports() { ApplicationReport.newInstance(applicationId4, ApplicationAttemptId.newInstance(applicationId4, 4), "user4", "queue4", "appname4", "host4", 127, null, - YarnApplicationState.FAILED, "diagnostics4", "url4", 4, 4, + YarnApplicationState.FAILED, "diagnostics4", "url4", 4, 4, 4, FinalApplicationStatus.SUCCEEDED, null, "N/A", 0.83789f, "NON-MAPREDUCE", null); applicationReports.add(newApplicationReport4); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/api/impl/TestYarnClient.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/api/impl/TestYarnClient.java index 41ef404be6b..813061e2af7 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/api/impl/TestYarnClient.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/api/impl/TestYarnClient.java @@ -752,7 +752,7 @@ public void setYarnApplicationState(YarnApplicationState state) { ApplicationReport newApplicationReport = ApplicationReport.newInstance( applicationId, ApplicationAttemptId.newInstance(applicationId, 1), "user", "queue", "appname", "host", 124, null, - YarnApplicationState.RUNNING, "diagnostics", "url", 0, 0, + YarnApplicationState.RUNNING, "diagnostics", "url", 0, 0, 0, FinalApplicationStatus.SUCCEEDED, null, "N/A", 0.53789f, "YARN", null); List applicationReports = new ArrayList(); applicationReports.add(newApplicationReport); @@ -827,7 +827,7 @@ public void setYarnApplicationState(YarnApplicationState state) { ApplicationReport newApplicationReport2 = ApplicationReport.newInstance( applicationId2, ApplicationAttemptId.newInstance(applicationId2, 2), "user2", "queue2", "appname2", "host2", 125, null, - YarnApplicationState.FINISHED, "diagnostics2", "url2", 2, 2, + YarnApplicationState.FINISHED, "diagnostics2", "url2", 2, 2, 2, FinalApplicationStatus.SUCCEEDED, null, "N/A", 0.63789f, "NON-YARN", null); applicationReports.add(newApplicationReport2); @@ -836,7 +836,7 @@ public void setYarnApplicationState(YarnApplicationState state) { ApplicationReport newApplicationReport3 = ApplicationReport.newInstance( applicationId3, ApplicationAttemptId.newInstance(applicationId3, 3), "user3", "queue3", "appname3", "host3", 126, null, - YarnApplicationState.RUNNING, "diagnostics3", "url3", 3, 3, + YarnApplicationState.RUNNING, "diagnostics3", "url3", 3, 3, 3, FinalApplicationStatus.SUCCEEDED, null, "N/A", 0.73789f, "MAPREDUCE", null); applicationReports.add(newApplicationReport3); @@ -847,7 +847,7 @@ public void setYarnApplicationState(YarnApplicationState state) { applicationId4, ApplicationAttemptId.newInstance(applicationId4, 4), "user4", "queue4", "appname4", "host4", 127, null, - YarnApplicationState.FAILED, "diagnostics4", "url4", 4, 4, + YarnApplicationState.FAILED, "diagnostics4", "url4", 4, 4, 4, FinalApplicationStatus.SUCCEEDED, null, "N/A", 0.83789f, "NON-MAPREDUCE", null); applicationReports.add(newApplicationReport4); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/cli/TestYarnCLI.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/cli/TestYarnCLI.java index 3c35b9cd313..4989c5a1a5b 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/cli/TestYarnCLI.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/cli/TestYarnCLI.java @@ -123,7 +123,7 @@ public void testGetApplicationReport() throws Exception { ApplicationReport newApplicationReport = ApplicationReport.newInstance( applicationId, ApplicationAttemptId.newInstance(applicationId, 1), "user", "queue", "appname", "host", 124, null, - YarnApplicationState.FINISHED, "diagnostics", "url", 0, 0, + YarnApplicationState.FINISHED, "diagnostics", "url", 0, 0, 0, FinalApplicationStatus.SUCCEEDED, usageReport, "N/A", 0.53789f, "YARN", null, null, false, Priority.newInstance(0), "high-mem", "high-mem"); newApplicationReport.setLogAggregationStatus(LogAggregationStatus.SUCCEEDED); @@ -370,7 +370,7 @@ public void testGetApplications() throws Exception { ApplicationReport newApplicationReport = ApplicationReport.newInstance( applicationId, ApplicationAttemptId.newInstance(applicationId, 1), "user", "queue", "appname", "host", 124, null, - YarnApplicationState.RUNNING, "diagnostics", "url", 0, 0, + YarnApplicationState.RUNNING, "diagnostics", "url", 0, 0, 0, FinalApplicationStatus.SUCCEEDED, null, "N/A", 0.53789f, "YARN", null, Sets.newHashSet("tag1", "tag3"), false, Priority.UNDEFINED, "", ""); List applicationReports = @@ -381,7 +381,7 @@ public void testGetApplications() throws Exception { ApplicationReport newApplicationReport2 = ApplicationReport.newInstance( applicationId2, ApplicationAttemptId.newInstance(applicationId2, 2), "user2", "queue2", "appname2", "host2", 125, null, - YarnApplicationState.FINISHED, "diagnostics2", "url2", 2, 2, + YarnApplicationState.FINISHED, "diagnostics2", "url2", 2, 2, 2, FinalApplicationStatus.SUCCEEDED, null, "N/A", 0.63789f, "NON-YARN", null, Sets.newHashSet("tag2", "tag3"), false, Priority.UNDEFINED, "", ""); @@ -391,7 +391,7 @@ public void testGetApplications() throws Exception { ApplicationReport newApplicationReport3 = ApplicationReport.newInstance( applicationId3, ApplicationAttemptId.newInstance(applicationId3, 3), "user3", "queue3", "appname3", "host3", 126, null, - YarnApplicationState.RUNNING, "diagnostics3", "url3", 3, 3, + YarnApplicationState.RUNNING, "diagnostics3", "url3", 3, 3, 3, FinalApplicationStatus.SUCCEEDED, null, "N/A", 0.73789f, "MAPREDUCE", null, Sets.newHashSet("tag1", "tag4"), false, Priority.UNDEFINED, "", ""); @@ -401,7 +401,7 @@ public void testGetApplications() throws Exception { ApplicationReport newApplicationReport4 = ApplicationReport.newInstance( applicationId4, ApplicationAttemptId.newInstance(applicationId4, 4), "user4", "queue4", "appname4", "host4", 127, null, - YarnApplicationState.FAILED, "diagnostics4", "url4", 4, 4, + YarnApplicationState.FAILED, "diagnostics4", "url4", 4, 4, 4, FinalApplicationStatus.SUCCEEDED, null, "N/A", 0.83789f, "NON-MAPREDUCE", null, Sets.newHashSet("tag1"), false, Priority.UNDEFINED, "", ""); @@ -411,7 +411,7 @@ public void testGetApplications() throws Exception { ApplicationReport newApplicationReport5 = ApplicationReport.newInstance( applicationId5, ApplicationAttemptId.newInstance(applicationId5, 5), "user5", "queue5", "appname5", "host5", 128, null, - YarnApplicationState.ACCEPTED, "diagnostics5", "url5", 5, 5, + YarnApplicationState.ACCEPTED, "diagnostics5", "url5", 5, 5, 5, FinalApplicationStatus.KILLED, null, "N/A", 0.93789f, "HIVE", null, Sets.newHashSet("tag2", "tag4"), false, Priority.UNDEFINED, "", ""); applicationReports.add(newApplicationReport5); @@ -420,7 +420,7 @@ public void testGetApplications() throws Exception { ApplicationReport newApplicationReport6 = ApplicationReport.newInstance( applicationId6, ApplicationAttemptId.newInstance(applicationId6, 6), "user6", "queue6", "appname6", "host6", 129, null, - YarnApplicationState.SUBMITTED, "diagnostics6", "url6", 6, 6, + YarnApplicationState.SUBMITTED, "diagnostics6", "url6", 6, 6, 6, FinalApplicationStatus.KILLED, null, "N/A", 0.99789f, "PIG", null, new HashSet(), false, Priority.UNDEFINED, "", ""); applicationReports.add(newApplicationReport6); @@ -994,7 +994,7 @@ public void testKillApplication() throws Exception { ApplicationReport newApplicationReport2 = ApplicationReport.newInstance( applicationId, ApplicationAttemptId.newInstance(applicationId, 1), "user", "queue", "appname", "host", 124, null, - YarnApplicationState.FINISHED, "diagnostics", "url", 0, 0, + YarnApplicationState.FINISHED, "diagnostics", "url", 0, 0, 0, FinalApplicationStatus.SUCCEEDED, null, "N/A", 0.53789f, "YARN", null); when(client.getApplicationReport(any(ApplicationId.class))).thenReturn( newApplicationReport2); @@ -1007,7 +1007,7 @@ public void testKillApplication() throws Exception { ApplicationReport newApplicationReport = ApplicationReport.newInstance( applicationId, ApplicationAttemptId.newInstance(applicationId, 1), "user", "queue", "appname", "host", 124, null, - YarnApplicationState.RUNNING, "diagnostics", "url", 0, 0, + YarnApplicationState.RUNNING, "diagnostics", "url", 0, 0, 0, FinalApplicationStatus.SUCCEEDED, null, "N/A", 0.53789f, "YARN", null); when(client.getApplicationReport(any(ApplicationId.class))).thenReturn( newApplicationReport); @@ -1046,12 +1046,12 @@ public void testKillApplications() throws Exception { ApplicationReport newApplicationReport1 = ApplicationReport.newInstance( applicationId1, ApplicationAttemptId.newInstance(applicationId1, 1), "user", "queue", "appname", "host", 124, null, - YarnApplicationState.FINISHED, "diagnostics", "url", 0, 0, + YarnApplicationState.FINISHED, "diagnostics", "url", 0, 0, 0, FinalApplicationStatus.SUCCEEDED, null, "N/A", 0.53789f, "YARN", null); ApplicationReport newApplicationReport2 = ApplicationReport.newInstance( applicationId2, ApplicationAttemptId.newInstance(applicationId2, 1), "user", "queue", "appname", "host", 124, null, - YarnApplicationState.FINISHED, "diagnostics", "url", 0, 0, + YarnApplicationState.FINISHED, "diagnostics", "url", 0, 0, 0, FinalApplicationStatus.SUCCEEDED, null, "N/A", 0.34344f, "YARN", null); when(client.getApplicationReport(applicationId1)).thenReturn( newApplicationReport1); @@ -1071,12 +1071,12 @@ public void testKillApplications() throws Exception { ApplicationReport newApplicationReport3 = ApplicationReport.newInstance( applicationId1, ApplicationAttemptId.newInstance(applicationId1, 1), "user", "queue", "appname", "host", 124, null, - YarnApplicationState.RUNNING, "diagnostics", "url", 0, 0, + YarnApplicationState.RUNNING, "diagnostics", "url", 0, 0, 0, FinalApplicationStatus.SUCCEEDED, null, "N/A", 0.53789f, "YARN", null); ApplicationReport newApplicationReport4 = ApplicationReport.newInstance( applicationId2, ApplicationAttemptId.newInstance(applicationId2, 1), "user", "queue", "appname", "host", 124, null, - YarnApplicationState.RUNNING, "diagnostics", "url", 0, 0, + YarnApplicationState.RUNNING, "diagnostics", "url", 0, 0, 0, FinalApplicationStatus.SUCCEEDED, null, "N/A", 0.53345f, "YARN", null); when(client.getApplicationReport(applicationId1)).thenReturn( newApplicationReport3); @@ -1114,7 +1114,7 @@ public void testKillApplications() throws Exception { ApplicationReport newApplicationReport5 = ApplicationReport.newInstance( applicationId1, ApplicationAttemptId.newInstance(applicationId1, 1), "user", "queue", "appname", "host", 124, null, - YarnApplicationState.RUNNING, "diagnostics", "url", 0, 0, + YarnApplicationState.RUNNING, "diagnostics", "url", 0, 0, 0, FinalApplicationStatus.SUCCEEDED, null, "N/A", 0.53345f, "YARN", null); when(client.getApplicationReport(applicationId1)).thenReturn( newApplicationReport5); @@ -1141,12 +1141,12 @@ public void testKillApplicationsOfDifferentEndStates() throws Exception { ApplicationReport newApplicationReport5 = ApplicationReport.newInstance( applicationId1, ApplicationAttemptId.newInstance(applicationId1, 1), "user", "queue", "appname", "host", 124, null, - YarnApplicationState.FINISHED, "diagnostics", "url", 0, 0, + YarnApplicationState.FINISHED, "diagnostics", "url", 0, 0, 0, FinalApplicationStatus.SUCCEEDED, null, "N/A", 0.53789f, "YARN", null); ApplicationReport newApplicationReport6 = ApplicationReport.newInstance( applicationId2, ApplicationAttemptId.newInstance(applicationId2, 1), "user", "queue", "appname", "host", 124, null, - YarnApplicationState.RUNNING, "diagnostics", "url", 0, 0, + YarnApplicationState.RUNNING, "diagnostics", "url", 0, 0, 0, FinalApplicationStatus.SUCCEEDED, null, "N/A", 0.53345f, "YARN", null); when(client.getApplicationReport(applicationId1)).thenReturn( newApplicationReport5); @@ -1169,7 +1169,7 @@ public void testMoveApplicationAcrossQueues() throws Exception { ApplicationReport newApplicationReport2 = ApplicationReport.newInstance( applicationId, ApplicationAttemptId.newInstance(applicationId, 1), "user", "queue", "appname", "host", 124, null, - YarnApplicationState.FINISHED, "diagnostics", "url", 0, 0, + YarnApplicationState.FINISHED, "diagnostics", "url", 0, 0, 0, FinalApplicationStatus.SUCCEEDED, null, "N/A", 0.53789f, "YARN", null); when(client.getApplicationReport(any(ApplicationId.class))).thenReturn( newApplicationReport2); @@ -1184,7 +1184,7 @@ public void testMoveApplicationAcrossQueues() throws Exception { ApplicationReport newApplicationReport = ApplicationReport.newInstance( applicationId, ApplicationAttemptId.newInstance(applicationId, 1), "user", "queue", "appname", "host", 124, null, - YarnApplicationState.RUNNING, "diagnostics", "url", 0, 0, + YarnApplicationState.RUNNING, "diagnostics", "url", 0, 0, 0, FinalApplicationStatus.SUCCEEDED, null, "N/A", 0.53789f, "YARN", null); when(client.getApplicationReport(any(ApplicationId.class))).thenReturn( newApplicationReport); @@ -1219,7 +1219,7 @@ public void testMoveApplicationAcrossQueuesWithNewCommand() throws Exception { ApplicationReport newApplicationReport2 = ApplicationReport.newInstance( applicationId, ApplicationAttemptId.newInstance(applicationId, 1), "user", "queue", "appname", "host", 124, null, - YarnApplicationState.FINISHED, "diagnostics", "url", 0, 0, + YarnApplicationState.FINISHED, "diagnostics", "url", 0, 0, 0, FinalApplicationStatus.SUCCEEDED, null, "N/A", 0.53789f, "YARN", null); when(client.getApplicationReport(any(ApplicationId.class))) .thenReturn(newApplicationReport2); @@ -1234,7 +1234,7 @@ public void testMoveApplicationAcrossQueuesWithNewCommand() throws Exception { ApplicationReport newApplicationReport = ApplicationReport.newInstance( applicationId, ApplicationAttemptId.newInstance(applicationId, 1), "user", "queue", "appname", "host", 124, null, - YarnApplicationState.RUNNING, "diagnostics", "url", 0, 0, + YarnApplicationState.RUNNING, "diagnostics", "url", 0, 0, 0, FinalApplicationStatus.SUCCEEDED, null, "N/A", 0.53789f, "YARN", null); when(client.getApplicationReport(any(ApplicationId.class))) .thenReturn(newApplicationReport); @@ -1922,7 +1922,7 @@ public void testUpdateApplicationPriority() throws Exception { ApplicationReport.newInstance(applicationId, ApplicationAttemptId.newInstance(applicationId, 1), "user", "queue", "appname", "host", 124, null, - YarnApplicationState.RUNNING, "diagnostics", "url", 0, 0, + YarnApplicationState.RUNNING, "diagnostics", "url", 0, 0, 0, FinalApplicationStatus.UNDEFINED, null, "N/A", 0.53789f, "YARN", null); when(client.getApplicationReport(any(ApplicationId.class))).thenReturn( @@ -2151,7 +2151,7 @@ public void testUpdateApplicationTimeout() throws Exception { ApplicationReport appReport = ApplicationReport.newInstance(applicationId, ApplicationAttemptId.newInstance(applicationId, 1), "user", "queue", "appname", "host", 124, null, YarnApplicationState.RUNNING, - "diagnostics", "url", 0, 0, FinalApplicationStatus.UNDEFINED, null, + "diagnostics", "url", 0, 0, 0, FinalApplicationStatus.UNDEFINED, null, "N/A", 0.53789f, "YARN", null); ApplicationTimeout timeout = ApplicationTimeout .newInstance(ApplicationTimeoutType.LIFETIME, "N/A", -1); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ApplicationReportPBImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ApplicationReportPBImpl.java index 036fa907fe6..cfb959d0fac 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ApplicationReportPBImpl.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ApplicationReportPBImpl.java @@ -221,6 +221,12 @@ public long getStartTime() { } @Override + public long getLaunchTime() { + ApplicationReportProtoOrBuilder p = viaProto ? proto : builder; + return p.getLaunchTime(); + } + + @Override public long getFinishTime() { ApplicationReportProtoOrBuilder p = viaProto ? proto : builder; return p.getFinishTime(); @@ -414,6 +420,12 @@ public void setStartTime(long startTime) { } @Override + public void setLaunchTime(long launchTime) { + maybeInitBuilder(); + builder.setLaunchTime(launchTime); + } + + @Override public void setFinishTime(long finishTime) { maybeInitBuilder(); builder.setFinishTime(finishTime); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/api/TestApplicatonReport.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/api/TestApplicatonReport.java index 46fc4d58d92..ea39a4ccdba 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/api/TestApplicatonReport.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/api/TestApplicatonReport.java @@ -58,7 +58,7 @@ protected static ApplicationReport createApplicationReport( ApplicationReport appReport = ApplicationReport.newInstance(appId, appAttemptId, "user", "queue", "appname", "host", 124, null, YarnApplicationState.FINISHED, - "diagnostics", "url", 0, 0, FinalApplicationStatus.SUCCEEDED, null, + "diagnostics", "url", 0, 0, 0, FinalApplicationStatus.SUCCEEDED, null, "N/A", 0.53789f, YarnConfiguration.DEFAULT_APPLICATION_TYPE, null, null, false, Priority.newInstance(0),"",""); return appReport; diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/ApplicationHistoryManagerImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/ApplicationHistoryManagerImpl.java index b8931d81abc..40702fb0e28 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/ApplicationHistoryManagerImpl.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/ApplicationHistoryManagerImpl.java @@ -137,9 +137,9 @@ private ApplicationReport convertToApplicationReport( currentApplicationAttemptId, appHistory.getUser(), appHistory.getQueue(), appHistory.getApplicationName(), host, rpcPort, null, appHistory.getYarnApplicationState(), appHistory.getDiagnosticsInfo(), - trackingUrl, appHistory.getStartTime(), appHistory.getFinishTime(), - appHistory.getFinalApplicationStatus(), null, "", 100, - appHistory.getApplicationType(), null); + trackingUrl, appHistory.getStartTime(), appHistory.getLaunchTime(), + appHistory.getFinishTime(), appHistory.getFinalApplicationStatus(), + null, "", 100, appHistory.getApplicationType(), null); } private ApplicationAttemptHistoryData getLastAttempt(ApplicationId appId) diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/ApplicationHistoryManagerOnTimelineStore.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/ApplicationHistoryManagerOnTimelineStore.java index 9240ed872e0..9d7d4d016e8 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/ApplicationHistoryManagerOnTimelineStore.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/ApplicationHistoryManagerOnTimelineStore.java @@ -249,6 +249,7 @@ private static ApplicationReportExt convertToApplicationReport( String type = null; boolean unmanagedApplication = false; long createdTime = 0; + long launchTime = 0; long finishedTime = 0; float progress = 0.0f; int applicationPriority = 0; @@ -280,10 +281,11 @@ private static ApplicationReportExt convertToApplicationReport( return new ApplicationReportExt(ApplicationReport.newInstance( ApplicationId.fromString(entity.getEntityId()), latestApplicationAttemptId, user, queue, name, null, -1, null, - state, diagnosticsInfo, null, createdTime, finishedTime, - finalStatus, null, null, progress, type, null, appTags, - unmanagedApplication, Priority.newInstance(applicationPriority), - appNodeLabelExpression, amNodeLabelExpression), appViewACLs); + state, diagnosticsInfo, null, createdTime, launchTime, + finishedTime, finalStatus, null, null, progress, type, null, + appTags, unmanagedApplication, + Priority.newInstance(applicationPriority), appNodeLabelExpression, + amNodeLabelExpression), appViewACLs); } if (entityInfo.containsKey(ApplicationMetricsConstants.QUEUE_ENTITY_INFO)) { queue = @@ -438,10 +440,10 @@ private static ApplicationReportExt convertToApplicationReport( return new ApplicationReportExt(ApplicationReport.newInstance( ApplicationId.fromString(entity.getEntityId()), latestApplicationAttemptId, user, queue, name, null, -1, null, state, - diagnosticsInfo, null, createdTime, finishedTime, finalStatus, - appResources, null, progress, type, null, appTags, unmanagedApplication, - Priority.newInstance(applicationPriority), appNodeLabelExpression, - amNodeLabelExpression), appViewACLs); + diagnosticsInfo, null, createdTime, launchTime, finishedTime, + finalStatus, appResources, null, progress, type, null, appTags, + unmanagedApplication, Priority.newInstance(applicationPriority), + appNodeLabelExpression, amNodeLabelExpression), appViewACLs); } private static long parseLong(Map entityInfo, diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/FileSystemApplicationHistoryStore.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/FileSystemApplicationHistoryStore.java index fa2da44f5f2..519250d0b39 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/FileSystemApplicationHistoryStore.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/FileSystemApplicationHistoryStore.java @@ -155,8 +155,8 @@ public ApplicationHistoryData getApplication(ApplicationId appId) boolean readFinishData = false; ApplicationHistoryData historyData = ApplicationHistoryData.newInstance(appId, null, null, null, null, - Long.MIN_VALUE, Long.MIN_VALUE, Long.MAX_VALUE, null, - FinalApplicationStatus.UNDEFINED, null); + Long.MIN_VALUE, Long.MIN_VALUE, Long.MAX_VALUE, Long.MAX_VALUE, + null, FinalApplicationStatus.UNDEFINED, null); while ((!readStartData || !readFinishData) && hfReader.hasNext()) { HistoryFileReader.Entry entry = hfReader.next(); if (entry.key.id.equals(appId.toString())) { @@ -600,6 +600,7 @@ private static void mergeApplicationHistoryData( private static void mergeApplicationHistoryData( ApplicationHistoryData historyData, ApplicationFinishData finishData) { + historyData.setLaunchTime(finishData.getLaunchTime()); historyData.setFinishTime(finishData.getFinishTime()); historyData.setDiagnosticsInfo(finishData.getDiagnosticsInfo()); historyData.setFinalApplicationStatus(finishData diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/MemoryApplicationHistoryStore.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/MemoryApplicationHistoryStore.java index c226ad3c90e..d6d20e0ed05 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/MemoryApplicationHistoryStore.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/MemoryApplicationHistoryStore.java @@ -142,7 +142,8 @@ public void applicationStarted(ApplicationStartData appStart) ApplicationHistoryData.newInstance(appStart.getApplicationId(), appStart.getApplicationName(), appStart.getApplicationType(), appStart.getQueue(), appStart.getUser(), appStart.getSubmitTime(), - appStart.getStartTime(), Long.MAX_VALUE, null, null, null)); + appStart.getStartTime(), Long.MAX_VALUE, Long.MAX_VALUE, + null, null, null)); if (oldData != null) { throw new IOException("The start information of application " + appStart.getApplicationId() + " is already stored."); @@ -165,6 +166,7 @@ public void applicationFinished(ApplicationFinishData appFinish) throw new IOException("The finish information of application " + appFinish.getApplicationId() + " is already stored."); } + data.setLaunchTime(appFinish.getLaunchTime()); data.setFinishTime(appFinish.getFinishTime()); data.setDiagnosticsInfo(appFinish.getDiagnosticsInfo()); data.setFinalApplicationStatus(appFinish.getFinalApplicationStatus()); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/records/ApplicationFinishData.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/records/ApplicationFinishData.java index 997fa6cbfd8..e926bc4d77c 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/records/ApplicationFinishData.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/records/ApplicationFinishData.java @@ -36,12 +36,13 @@ @Public @Unstable public static ApplicationFinishData newInstance(ApplicationId applicationId, - long finishTime, String diagnosticsInfo, + long launchTime, long finishTime, String diagnosticsInfo, FinalApplicationStatus finalApplicationStatus, YarnApplicationState yarnApplicationState) { ApplicationFinishData appFD = Records.newRecord(ApplicationFinishData.class); appFD.setApplicationId(applicationId); + appFD.setLaunchTime(launchTime); appFD.setFinishTime(finishTime); appFD.setDiagnosticsInfo(diagnosticsInfo); appFD.setFinalApplicationStatus(finalApplicationStatus); @@ -59,6 +60,14 @@ public static ApplicationFinishData newInstance(ApplicationId applicationId, @Public @Unstable + public abstract long getLaunchTime(); + + @Public + @Unstable + public abstract void setLaunchTime(long launchTime); + + @Public + @Unstable public abstract long getFinishTime(); @Public diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/records/ApplicationHistoryData.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/records/ApplicationHistoryData.java index b7d16f3c412..6be83311e25 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/records/ApplicationHistoryData.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/records/ApplicationHistoryData.java @@ -46,6 +46,8 @@ private long startTime; + private long launchTime; + private long finishTime; private String diagnosticsInfo; @@ -58,8 +60,9 @@ @Unstable public static ApplicationHistoryData newInstance(ApplicationId applicationId, String applicationName, String applicationType, String queue, - String user, long submitTime, long startTime, long finishTime, - String diagnosticsInfo, FinalApplicationStatus finalApplicationStatus, + String user, long submitTime, long startTime, long launchTime, + long finishTime, String diagnosticsInfo, + FinalApplicationStatus finalApplicationStatus, YarnApplicationState yarnApplicationState) { ApplicationHistoryData appHD = new ApplicationHistoryData(); appHD.setApplicationId(applicationId); @@ -69,6 +72,7 @@ public static ApplicationHistoryData newInstance(ApplicationId applicationId, appHD.setUser(user); appHD.setSubmitTime(submitTime); appHD.setStartTime(startTime); + appHD.setLaunchTime(launchTime); appHD.setFinishTime(finishTime); appHD.setDiagnosticsInfo(diagnosticsInfo); appHD.setFinalApplicationStatus(finalApplicationStatus); @@ -162,6 +166,18 @@ public void setStartTime(long startTime) { @Public @Unstable + public long getLaunchTime() { + return launchTime; + } + + @Public + @Unstable + public void setLaunchTime(long launchTime) { + this.launchTime = launchTime; + } + + @Public + @Unstable public long getFinishTime() { return finishTime; } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/records/impl/pb/ApplicationFinishDataPBImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/records/impl/pb/ApplicationFinishDataPBImpl.java index 337426d0284..aad78b1b05c 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/records/impl/pb/ApplicationFinishDataPBImpl.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/records/impl/pb/ApplicationFinishDataPBImpl.java @@ -73,6 +73,18 @@ public void setApplicationId(ApplicationId applicationId) { } @Override + public long getLaunchTime() { + ApplicationFinishDataProtoOrBuilder p = viaProto ? proto : builder; + return p.getLaunchTime(); + } + + @Override + public void setLaunchTime(long launchTime) { + maybeInitBuilder(); + builder.setLaunchTime(launchTime); + } + + @Override public long getFinishTime() { ApplicationFinishDataProtoOrBuilder p = viaProto ? proto : builder; return p.getFinishTime(); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/ApplicationHistoryStoreTestUtils.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/ApplicationHistoryStoreTestUtils.java index de4051a494c..98f5b65d08f 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/ApplicationHistoryStoreTestUtils.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/ApplicationHistoryStoreTestUtils.java @@ -49,7 +49,7 @@ protected void writeApplicationStartData(ApplicationId appId) protected void writeApplicationFinishData(ApplicationId appId) throws IOException { - store.applicationFinished(ApplicationFinishData.newInstance(appId, 0, + store.applicationFinished(ApplicationFinishData.newInstance(appId, 0, 0, appId.toString(), FinalApplicationStatus.UNDEFINED, YarnApplicationState.FINISHED)); } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/utils/BuilderUtils.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/utils/BuilderUtils.java index e7f47af2647..708bfcfccd4 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/utils/BuilderUtils.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/utils/BuilderUtils.java @@ -387,7 +387,7 @@ public static ApplicationReport newApplicationReport( ApplicationId applicationId, ApplicationAttemptId applicationAttemptId, String user, String queue, String name, String host, int rpcPort, Token clientToAMToken, YarnApplicationState state, String diagnostics, - String url, long startTime, long finishTime, + String url, long startTime, long launchTime, long finishTime, FinalApplicationStatus finalStatus, ApplicationResourceUsageReport appResources, String origTrackingUrl, float progress, String appType, Token amRmToken, Set tags, @@ -406,6 +406,7 @@ public static ApplicationReport newApplicationReport( report.setDiagnostics(diagnostics); report.setTrackingUrl(url); report.setStartTime(startTime); + report.setLaunchTime(launchTime); report.setFinishTime(finishTime); report.setFinalApplicationStatus(finalStatus); report.setApplicationResourceUsageReport(appResources); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/webapp/AppBlock.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/webapp/AppBlock.java index 693aa046df5..e0f93f101c0 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/webapp/AppBlock.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/webapp/AppBlock.java @@ -233,10 +233,10 @@ private void generateOverviewTable(AppInfo app, String schedulerPath, .__("FinalStatus Reported by AM:", clairfyAppFinalStatus(app.getFinalAppStatus())) .__("Started:", Times.format(app.getStartedTime())) - .__( - "Elapsed:", - StringUtils.formatTime(Times.elapsed(app.getStartedTime(), - app.getFinishedTime()))) + .__("Pending:", StringUtils.formatTime(app.getPendingTime())) + .__("Launched:", Times.format(app.getLaunchTime())) + .__("Elapsed:", StringUtils.formatTime(app.getElapsedTime())) + .__("Finished:", Times.format(app.getFinishedTime())) .__( "Tracking URL:", app.getTrackingUrl() == null diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/webapp/AppsBlock.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/webapp/AppsBlock.java index d836e641177..0fb7fb2b0dd 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/webapp/AppsBlock.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/webapp/AppsBlock.java @@ -145,7 +145,9 @@ protected void renderData(Block html) { html.table("#apps").thead().tr().th(".id", "ID").th(".user", "User") .th(".name", "Name").th(".type", "Application Type") .th(".queue", "Queue").th(".priority", "Application Priority") - .th(".starttime", "StartTime").th(".finishtime", "FinishTime") + .th(".starttime", "StartTime") + .th(".launchtime", "LaunchTime") + .th(".finishtime", "FinishTime") .th(".state", "State").th(".finalstatus", "FinalStatus") .th(".progress", "Progress").th(".ui", "Tracking UI").__().__().tbody(); @@ -183,6 +185,7 @@ protected void renderData(Block html) { .getQueue()))).append("\",\"").append(String .valueOf(app.getPriority())) .append("\",\"").append(app.getStartedTime()) + .append("\",\"").append(app.getLaunchTime()) .append("\",\"").append(app.getFinishedTime()) .append("\",\"") .append(app.getAppState() == null ? UNAVAILABLE : app.getAppState()) diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/webapp/WebPageUtils.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/webapp/WebPageUtils.java index 3a26ae58890..7f2d71401b7 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/webapp/WebPageUtils.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/webapp/WebPageUtils.java @@ -51,7 +51,7 @@ private static String getAppsTableColumnDefs( sb.append("[\n") .append("{'sType':'natural', 'aTargets': [0]") .append(", 'mRender': parseHadoopID }") - .append("\n, {'sType':'numeric', 'aTargets': [6, 7]") + .append("\n, {'sType':'numeric', 'aTargets': [6, 7, 8]") .append(", 'mRender': renderHadoopDate }") .append("\n, {'sType':'numeric', bSearchable:false, 'aTargets':"); if (isFairSchedulerPage) { diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/webapp/dao/AppInfo.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/webapp/dao/AppInfo.java index ac2f8da6aa2..a4676083e07 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/webapp/dao/AppInfo.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/webapp/dao/AppInfo.java @@ -55,7 +55,9 @@ protected FinalApplicationStatus finalAppStatus; protected long submittedTime; protected long startedTime; + protected long launchTime; protected long finishedTime; + protected long pendingTime; protected long elapsedTime; protected String applicationTags; protected int priority; @@ -88,8 +90,10 @@ public AppInfo(ApplicationReport app) { originalTrackingUrl = app.getOriginalTrackingUrl(); submittedTime = app.getStartTime(); startedTime = app.getStartTime(); + launchTime = app.getLaunchTime(); finishedTime = app.getFinishTime(); - elapsedTime = Times.elapsed(startedTime, finishedTime); + pendingTime = Times.elapsed(startedTime, launchTime); + elapsedTime = Times.elapsed(launchTime, finishedTime); finalAppStatus = app.getFinalApplicationStatus(); priority = 0; if (app.getPriority() != null) { @@ -202,10 +206,18 @@ public long getStartedTime() { return startedTime; } + public long getLaunchTime() { + return launchTime; + } + public long getFinishedTime() { return finishedTime; } + public long getPendingTime() { + return pendingTime; + } + public long getElapsedTime() { return elapsedTime; } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ahs/RMApplicationHistoryWriter.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ahs/RMApplicationHistoryWriter.java index 19c5c27f961..7339f5f9be5 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ahs/RMApplicationHistoryWriter.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ahs/RMApplicationHistoryWriter.java @@ -238,8 +238,8 @@ public void applicationFinished(RMApp app, RMAppState finalState) { dispatcher.getEventHandler().handle( new WritingApplicationFinishEvent(app.getApplicationId(), ApplicationFinishData.newInstance(app.getApplicationId(), - app.getFinishTime(), app.getDiagnostics().toString(), - app.getFinalApplicationStatus(), + app.getLaunchTime(), app.getFinishTime(), + app.getDiagnostics().toString(), app.getFinalApplicationStatus(), RMServerUtils.createApplicationState(finalState)))); } } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/RMApp.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/RMApp.java index 43fd1fbfbdf..93539e1b1f0 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/RMApp.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/RMApp.java @@ -172,7 +172,15 @@ ApplicationReport createAndGetApplicationReport(String clientUserName, * @return the submit time of the application. */ long getSubmitTime(); - + + /** + * The launch time of the application. + * Since getStartTime() returns what is essentially submit time, + * this new field is to prevent potential backwards compatibility issues. + * @return the launch time of the application. + */ + long getLaunchTime(); + /** * The tracking url for the application master. * @return the tracking url for the application master. diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/RMAppImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/RMAppImpl.java index 03be7937b3a..83d77763182 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/RMAppImpl.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/RMAppImpl.java @@ -161,6 +161,7 @@ // Mutable fields private long startTime; + private long launchTime = 0; private long finishTime = 0; private long storedFinishTime = 0; private int firstAttemptIdInStateStore = 1; @@ -761,9 +762,9 @@ public ApplicationReport createAndGetApplicationReport(String clientUserName, this.applicationId, currentApplicationAttemptId, this.user, this.queue, this.name, host, rpcPort, clientToAMToken, createApplicationState(), diags, trackingUrl, this.startTime, - this.finishTime, finishState, appUsageReport, origTrackingUrl, - progress, this.applicationType, amrmToken, applicationTags, - this.getApplicationPriority()); + this.launchTime, this.finishTime, finishState, appUsageReport, + origTrackingUrl, progress, this.applicationType, amrmToken, + applicationTags, this.getApplicationPriority()); report.setLogAggregationStatus(logAggregationStatus); report.setUnmanagedApp(submissionContext.getUnmanagedAM()); report.setAppNodeLabelExpression(getAppNodeLabelExpression()); @@ -831,6 +832,26 @@ public long getStartTime() { } @Override + public long getLaunchTime() { + this.readLock.lock(); + + try { + return this.launchTime; + } finally { + this.readLock.unlock(); + } + } + + public void setLaunchTime(long launchTime) { + this.writeLock.lock(); + try { + this.launchTime = launchTime; + } finally { + this.writeLock.unlock(); + } + } + + @Override public long getSubmitTime() { return this.submitTime; } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/RMAppAttemptImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/RMAppAttemptImpl.java index d748860d874..f7839270f12 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/RMAppAttemptImpl.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/RMAppAttemptImpl.java @@ -1526,6 +1526,10 @@ public void transition(RMAppAttemptImpl appAttempt, RMAppAttemptEvent event) { @Override public void transition(RMAppAttemptImpl appAttempt, RMAppAttemptEvent event) { + RMApp rmApp = appAttempt.rmApp; + if (rmApp.getLaunchTime() == 0) { + ((RMAppImpl) rmApp).setLaunchTime(System.currentTimeMillis()); + } if (event.getType() == RMAppAttemptEventType.LAUNCHED || event.getType() == RMAppAttemptEventType.REGISTERED) { appAttempt.launchAMEndTime = System.currentTimeMillis(); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/FairSchedulerAppsBlock.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/FairSchedulerAppsBlock.java index ac88f861feb..ff29f26b6da 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/FairSchedulerAppsBlock.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/FairSchedulerAppsBlock.java @@ -89,6 +89,7 @@ public FairSchedulerAppsBlock(ResourceManager rm, ViewContext ctx, th(".queue", "Queue"). th(".fairshare", "Fair Share"). th(".starttime", "StartTime"). + th(".launchtime", "LaunchTime"). th(".finishtime", "FinishTime"). th(".state", "State"). th(".finalstatus", "FinalStatus"). @@ -135,6 +136,7 @@ public FairSchedulerAppsBlock(ResourceManager rm, ViewContext ctx, appInfo.getQueue()))).append("\",\"") .append(fairShare).append("\",\"") .append(appInfo.getStartTime()).append("\",\"") + .append(appInfo.getLaunchTime()).append("\",\"") .append(appInfo.getFinishTime()).append("\",\"") .append(appInfo.getState()).append("\",\"") .append(appInfo.getFinalStatus()).append("\",\"") diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMAppsBlock.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMAppsBlock.java index ede71e34378..57c69a0b205 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMAppsBlock.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMAppsBlock.java @@ -60,6 +60,7 @@ protected void renderData(Block html) { .th(".name", "Name").th(".type", "Application Type") .th(".queue", "Queue").th(".priority", "Application Priority") .th(".starttime", "StartTime") + .th(".launchtime", "LaunchTime") .th(".finishtime", "FinishTime").th(".state", "State") .th(".finalstatus", "FinalStatus") .th(".runningcontainer", "Running Containers") @@ -132,6 +133,7 @@ protected void renderData(Block html) { .getQueue()))).append("\",\"").append(String .valueOf(app.getPriority())) .append("\",\"").append(app.getStartedTime()) + .append("\",\"").append(app.getLaunchTime()) .append("\",\"").append(app.getFinishedTime()) .append("\",\"") .append(app.getAppState() == null ? UNAVAILABLE : app.getAppState()) diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/AppInfo.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/AppInfo.java index 9fb8fb5858d..87bb4f678d9 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/AppInfo.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/AppInfo.java @@ -87,7 +87,9 @@ // these are only allowed if acls allow protected long startedTime; + protected long launchTime; protected long finishedTime; + protected long pendingTime; protected long elapsedTime; protected String amContainerLogs; protected String amHostHttpAddress; @@ -177,9 +179,12 @@ public AppInfo(ResourceManager rm, RMApp app, Boolean hasAccess, this.clusterId = ResourceManager.getClusterTimeStamp(); if (hasAccess) { this.startedTime = app.getStartTime(); + this.launchTime = app.getLaunchTime(); this.finishedTime = app.getFinishTime(); - this.elapsedTime = - Times.elapsed(app.getStartTime(), app.getFinishTime()); + this.pendingTime = Times.elapsed(app.getStartTime(), + app.getLaunchTime()); + this.elapsedTime = Times.elapsed(app.getLaunchTime(), + app.getFinishTime()); this.logAggregationStatus = app.getLogAggregationStatusForAppReport(); RMAppAttempt attempt = app.getCurrentAppAttempt(); if (attempt != null) { @@ -343,6 +348,10 @@ public String getTrackingUrlPretty() { return this.trackingUrlPretty; } + public long getLaunchTime() { + return this.launchTime; + } + public long getStartTime() { return this.startedTime; } @@ -351,6 +360,10 @@ public long getFinishTime() { return this.finishedTime; } + public long getPendingTime() { + return this.pendingTime; + } + public long getElapsedTime() { return this.elapsedTime; } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/applicationsmanager/MockAsm.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/applicationsmanager/MockAsm.java index 5246eb79a15..935b9930999 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/applicationsmanager/MockAsm.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/applicationsmanager/MockAsm.java @@ -87,7 +87,12 @@ public long getStartTime() { public long getSubmitTime() { throw new UnsupportedOperationException("Not supported yet."); } - + + @Override + public long getLaunchTime() { + throw new UnsupportedOperationException("Not supported yet."); + } + @Override public long getFinishTime() { throw new UnsupportedOperationException("Not supported yet."); @@ -259,6 +264,7 @@ public static RMApp newApplication(int i) { final String name = newAppName(); final String queue = newQueue(); final long start = 123456 + i * 1000; + final long launch = start + i * 100; final long finish = 234567 + i * 1000; final String type = YarnConfiguration.DEFAULT_APPLICATION_TYPE; YarnApplicationState[] allStates = YarnApplicationState.values(); @@ -295,6 +301,11 @@ public long getStartTime() { } @Override + public long getLaunchTime() { + return launch; + } + + @Override public long getFinishTime() { return finish; } @@ -343,9 +354,9 @@ public ApplicationReport createAndGetApplicationReport( ApplicationReport report = ApplicationReport.newInstance( getApplicationId(), appAttemptId, getUser(), getQueue(), getName(), null, 0, null, null, getDiagnostics().toString(), - getTrackingUrl(), getStartTime(), getFinishTime(), - getFinalApplicationStatus(), usageReport , null, getProgress(), - type, null); + getTrackingUrl(), getStartTime(), getLaunchTime(), + getFinishTime(), getFinalApplicationStatus(), usageReport, + null, getProgress(), type, null); return report; } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/MockRMApp.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/MockRMApp.java index 9290ff8faa0..7846ea510c9 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/MockRMApp.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/MockRMApp.java @@ -53,6 +53,7 @@ String name = MockApps.newAppName(); String queue = MockApps.newQueue(); long start = System.currentTimeMillis() - (int) (Math.random() * DT); + long launch = start; long submit = start - (int) (Math.random() * DT); long finish = 0; RMAppState state = RMAppState.NEW; @@ -183,6 +184,11 @@ public long getSubmitTime() { return submit; } + @Override + public long getLaunchTime() { + return launch; + } + public void setStartTime(long time) { this.start = time; } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebApp.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebApp.java index 8a6504002a1..e8b5668a2bc 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebApp.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebApp.java @@ -260,7 +260,7 @@ public static ClientRMService mockClientRMService(RMContext rmContext) { app.getName(), (String) null, 0, (Token) null, app.createApplicationState(), app.getDiagnostics().toString(), (String) null, - app.getStartTime(), app.getFinishTime(), + app.getStartTime(), app.getLaunchTime(), app.getFinishTime(), app.getFinalApplicationStatus(), (ApplicationResourceUsageReport) null, app.getTrackingUrl(), app.getProgress(), app.getApplicationType(), (Token) null); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesApps.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesApps.java index 1cbdec3a225..f0b0cb90850 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesApps.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesApps.java @@ -1478,7 +1478,9 @@ public void verifyAppsXML(NodeList nodes, RMApp app, boolean hasResourceReq) WebServicesTestUtils.getXmlString(element, "diagnostics"), WebServicesTestUtils.getXmlLong(element, "clusterId"), WebServicesTestUtils.getXmlLong(element, "startedTime"), + WebServicesTestUtils.getXmlLong(element, "launchTime"), WebServicesTestUtils.getXmlLong(element, "finishedTime"), + WebServicesTestUtils.getXmlLong(element, "pendingTime"), WebServicesTestUtils.getXmlLong(element, "elapsedTime"), WebServicesTestUtils.getXmlString(element, "amHostHttpAddress"), WebServicesTestUtils.getXmlString(element, "amContainerLogs"), @@ -1528,7 +1530,7 @@ public void verifyAppsXML(NodeList nodes, RMApp app, boolean hasResourceReq) public void verifyAppInfo(JSONObject info, RMApp app, boolean hasResourceReqs) throws JSONException, Exception { - int expectedNumberOfElements = 36 + (hasResourceReqs ? 2 : 0); + int expectedNumberOfElements = 38 + (hasResourceReqs ? 2 : 0); String appNodeLabelExpression = null; String amNodeLabelExpression = null; if (app.getApplicationSubmissionContext() @@ -1554,7 +1556,8 @@ public void verifyAppInfo(JSONObject info, RMApp app, boolean hasResourceReqs) info.getString("state"), info.getString("finalStatus"), (float) info.getDouble("progress"), info.getString("trackingUI"), info.getString("diagnostics"), info.getLong("clusterId"), - info.getLong("startedTime"), info.getLong("finishedTime"), + info.getLong("startedTime"), info.getLong("launchTime"), + info.getLong("finishedTime"), info.getLong("pendingTime"), info.getLong("elapsedTime"), info.getString("amHostHttpAddress"), info.getString("amContainerLogs"), info.getInt("allocatedMB"), info.getInt("allocatedVCores"), info.getInt("runningContainers"), @@ -1578,7 +1581,8 @@ public void verifyAppInfo(JSONObject info, RMApp app, boolean hasResourceReqs) public void verifyAppInfoGeneric(RMApp app, String id, String user, String name, String applicationType, String queue, int prioirty, String state, String finalStatus, float progress, String trackingUI, - String diagnostics, long clusterId, long startedTime, long finishedTime, + String diagnostics, long clusterId, long startedTime, + long launchTime, long finishedTime, long pendingTime, long elapsedTime, String amHostHttpAddress, String amContainerLogs, int allocatedMB, int allocatedVCores, int numContainers, float queueUsagePerc, float clusterUsagePerc, @@ -1610,9 +1614,12 @@ public void verifyAppInfoGeneric(RMApp app, String id, String user, assertEquals("clusterId doesn't match", ResourceManager.getClusterTimeStamp(), clusterId); assertEquals("startedTime doesn't match", app.getStartTime(), startedTime); + assertEquals("launchTime doesn't match", app.getLaunchTime(), + launchTime); assertEquals("finishedTime doesn't match", app.getFinishTime(), finishedTime); - assertTrue("elapsed time not greater than 0", elapsedTime > 0); + assertTrue("pending time not greater than 0", pendingTime > 0); + assertTrue("elapsed time not valid", elapsedTime >= 0); WebServicesTestUtils.checkStringMatch("amHostHttpAddress", app .getCurrentAppAttempt().getMasterContainer().getNodeHttpAddress(), amHostHttpAddress); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/ResourceManagerRest.md b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/ResourceManagerRest.md index 7c1f4188651..e092ed205d7 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/ResourceManagerRest.md +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/ResourceManagerRest.md @@ -1394,7 +1394,9 @@ Response Body: "applicationTags": "", "priority": -1, "startedTime": 1476913457320, + "launchTime": 1476913472749, "finishedTime": 1476913761898, + "pendingTime": 15429, "elapsedTime": 304578, "amContainerLogs": "http://host.domain.com:8042/node/containerlogs/container_1476912658570_0002_02_000001/user2", "amHostHttpAddress": "host.domain.com:8042", @@ -1470,7 +1472,9 @@ Response Body: "applicationTags": "", "priority": -1, "startedTime": 1476913464750, + "launchTime": 1476913494913, "finishedTime": 1476913863175, + "pendingTime": 30163, "elapsedTime": 398425, "amContainerLogs": "http://host.domain.com:8042/node/containerlogs/container_1476912658570_0001_02_000001/user1", "amHostHttpAddress": "host.domain.com:8042", @@ -1570,7 +1574,9 @@ Response Body: -1 1476913457320 + 1476913472749 1476913761898 + 15429 304578 http://host.domain.com:8042/node/containerlogs/container_1476912658570_0002_02_000001/user2 host.domain.com:8042 @@ -1645,7 +1651,9 @@ Response Body: -1 1476913464750 + 1476913494913 1476913863175 + 30163 398425 http://host.domain.com:8042/node/containerlogs/container_1476912658570_0001_02_000001/user1 host.domain.com:8042 @@ -1855,8 +1863,10 @@ Note that depending on security settings a user might not be able to see all the | applicationTags | string | Comma separated tags of an application | | priority | string | Priority of the submitted application | | startedTime | long | The time in which application started (in ms since epoch) | +| launchTime | long | The time at which the application launched (in ms since epoch) | | finishedTime | long | The time in which the application finished (in ms since epoch) | -| elapsedTime | long | The elapsed time since the application started (in ms) | +| pendingTime | long | The elapsed time between when the application was started and launched (in ms) | +| elapsedTime | long | The elapsed time since the application was launched (in ms) | | amContainerLogs | string | The URL of the application master container logs | | amHostHttpAddress | string | The nodes http address of the application master | | amRPCAddress | string | The RPC address of the application master | diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/controllers/app-table-columns.js b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/controllers/app-table-columns.js index 8a34f1a2715..290372760e2 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/controllers/app-table-columns.js +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/controllers/app-table-columns.js @@ -77,6 +77,18 @@ export default Ember.Controller.extend({ headerTitle: 'Start Time', contentPath: 'startTime', }, { + id: 'pendingTime', + headerTitle: 'Pending Time', + contentPath: 'pendingTime', + cellDefinition: { + type: "duration" + } + }, + { + id: 'launchTime', + headerTitle: 'Launch Time', + contentPath: 'validatedLaunchTs', + }, { id: 'elTime', headerTitle: 'Elapsed Time', contentPath: 'elapsedTime', diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/models/yarn-app.js b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/models/yarn-app.js index 47814e48c2b..a06f466a435 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/models/yarn-app.js +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/models/yarn-app.js @@ -25,6 +25,8 @@ export default DS.Model.extend({ queue: DS.attr('string'), state: DS.attr('string'), startTime: DS.attr('string'), + pendingTime: DS.attr('string'), + launchTime: DS.attr('string'), elapsedTime: DS.attr('string'), finalStatus: DS.attr('string'), finishedTime: DS.attr('finishedTime'), @@ -57,17 +59,28 @@ export default DS.Model.extend({ return this.get('finalStatus') === "FAILED"; }.property("finalStatus"), - validatedFinishedTs: function() { - if (this.get("finishedTime") < this.get("startTime")) { + formattedPendingTime: function() { + return Converter.msToElapsedTimeUnit(this.get('pendingTime')); + }.property('pendingTime'), + + validatedLaunchTs: function() { + if (this.get("launchTime") < this.get("startTime")) { return "N/A"; } - return this.get("finishedTime"); - }.property("finishedTime"), + return this.get("launchTime"); + }.property("launchTime"), formattedElapsedTime: function() { return Converter.msToElapsedTimeUnit(this.get('elapsedTime')); }.property('elapsedTime'), + validatedFinishedTs: function() { + if (this.get("finishedTime") < this.get("startTime")) { + return "N/A"; + } + return this.get("finishedTime"); + }.property("finishedTime"), + allocatedResource: function() { return Converter.resourceToString(this.get("allocatedMB"), this.get("allocatedVCores")); }.property("allocatedMB", "allocatedVCores"), diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/serializers/yarn-app.js b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/serializers/yarn-app.js index 1462f5aa085..47836537191 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/serializers/yarn-app.js +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/serializers/yarn-app.js @@ -43,6 +43,8 @@ export default DS.JSONAPISerializer.extend({ queue: payload.queue, state: payload.state, startTime: Converter.timeStampToDate(payload.startedTime), + pendingTime: payload.pendingTime, + launchTime: Converter.timeStampToDate(payload.launchTime), elapsedTime: payload.elapsedTime, finishedTime: Converter.timeStampToDate(payload.finishedTime), finalStatus: payload.finalStatus, diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/templates/components/app-table.hbs b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/templates/components/app-table.hbs index 3bad063b422..5ff6b89d10d 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/templates/components/app-table.hbs +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/templates/components/app-table.hbs @@ -27,6 +27,8 @@ State Final Status Start Time + Pending Time + Launch Time Elapsed Time Finished Time Priority @@ -46,6 +48,8 @@ {{app.state}} {{app.finalStatus}} {{app.startTime}} + {{app.pendingTime}} + {{app.validatedLaunchTs}} {{app.elapsedTime}} {{app.validatedFinishedTs}} {{app.priority}} @@ -69,6 +73,8 @@ {{app.state}} {{app.finalStatus}} {{app.startTime}} + {{app.pendingTime}} + {{app.validatedLaunchTs}} {{app.elapsedTime}} {{app.validatedFinishedTs}} {{app.priority}} diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/templates/yarn-app/info.hbs b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/templates/yarn-app/info.hbs index 3cfec338699..c0800a92db0 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/templates/yarn-app/info.hbs +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/templates/yarn-app/info.hbs @@ -40,6 +40,8 @@ State Final Status Start Time + Pending Time + Launch Time Elapsed Time Finished Time Priority @@ -63,6 +65,8 @@ {{model.app.startTime}} + {{model.app.formattedPendingTime}} + {{model.app.validatedLaunchTs}} {{model.app.formattedElapsedTime}} {{model.app.validatedFinishedTs}} {{model.app.priority}} -- 2.11.0 (Apple Git-81)