From fdee04a90a35be7de3ad8c911082ee417835d6bd Mon Sep 17 00:00:00 2001 From: Abdullah Yousufi Date: Mon, 28 Aug 2017 10:37:08 -0700 Subject: [PATCH] submit time changes Change-Id: Iaa2954a69b29e7d28f475d8cfaa489e8db6c2e84 --- .../org/apache/hadoop/mapred/NotRunningJob.java | 2 +- .../hadoop/mapred/TestClientServiceDelegate.java | 4 +- .../org/apache/hadoop/mapred/TestYARNRunner.java | 2 +- .../hadoop/yarn/api/records/ApplicationReport.java | 25 +++++++++--- .../proto/server/application_history_server.proto | 18 +++++---- .../src/main/proto/yarn_protos.proto | 31 +++++++-------- .../hadoop/yarn/client/ProtocolHATestBase.java | 2 +- .../hadoop/yarn/client/api/impl/TestAHSClient.java | 8 ++-- .../yarn/client/api/impl/TestYarnClient.java | 8 ++-- .../apache/hadoop/yarn/client/cli/TestYarnCLI.java | 44 +++++++++++----------- .../records/impl/pb/ApplicationReportPBImpl.java | 12 ++++++ .../hadoop/yarn/api/TestApplicatonReport.java | 3 +- .../ApplicationHistoryManagerImpl.java | 3 +- .../ApplicationHistoryManagerOnTimelineStore.java | 5 ++- .../FileSystemApplicationHistoryStore.java | 2 +- .../MemoryApplicationHistoryStore.java | 3 +- .../records/ApplicationFinishData.java | 11 +++++- .../records/ApplicationHistoryData.java | 17 ++++++++- .../impl/pb/ApplicationFinishDataPBImpl.java | 12 ++++++ .../ApplicationHistoryStoreTestUtils.java | 2 +- .../hadoop/yarn/server/utils/BuilderUtils.java | 3 +- .../apache/hadoop/yarn/server/webapp/AppBlock.java | 10 ++--- .../hadoop/yarn/server/webapp/AppsBlock.java | 4 +- .../hadoop/yarn/server/webapp/WebPageUtils.java | 2 +- .../hadoop/yarn/server/webapp/dao/AppInfo.java | 14 ++++++- .../ahs/RMApplicationHistoryWriter.java | 4 +- .../yarn/server/resourcemanager/rmapp/RMApp.java | 10 ++++- .../server/resourcemanager/rmapp/RMAppImpl.java | 19 ++++++++-- .../webapp/FairSchedulerAppsBlock.java | 2 + .../server/resourcemanager/webapp/RMAppsBlock.java | 2 + .../server/resourcemanager/webapp/dao/AppInfo.java | 15 +++++++- .../applicationsmanager/MockAsm.java | 15 +++++++- .../server/resourcemanager/rmapp/MockRMApp.java | 6 +++ .../resourcemanager/webapp/TestRMWebApp.java | 2 +- .../webapp/TestRMWebServicesApps.java | 16 +++++--- .../src/site/markdown/ResourceManagerRest.md | 12 +++++- .../webapp/app/controllers/app-table-columns.js | 14 ++++++- .../src/main/webapp/app/models/yarn-app.js | 21 +++++++++-- .../src/main/webapp/app/serializers/yarn-app.js | 2 + .../webapp/app/templates/components/app-table.hbs | 6 +++ .../main/webapp/app/templates/yarn-app/info.hbs | 4 ++ 41 files changed, 294 insertions(+), 103 deletions(-) diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/main/java/org/apache/hadoop/mapred/NotRunningJob.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/main/java/org/apache/hadoop/mapred/NotRunningJob.java index 90635a68854..bfdc6db8abd 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/main/java/org/apache/hadoop/mapred/NotRunningJob.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/main/java/org/apache/hadoop/mapred/NotRunningJob.java @@ -89,7 +89,7 @@ private ApplicationReport getUnknownApplicationReport() { // used for a non running job return ApplicationReport.newInstance(unknownAppId, unknownAttemptId, "N/A", "N/A", "N/A", "N/A", 0, null, YarnApplicationState.NEW, "N/A", - "N/A", 0, 0, FinalApplicationStatus.UNDEFINED, null, "N/A", 0.0f, + "N/A", 0, 0, 0, FinalApplicationStatus.UNDEFINED, null, "N/A", 0.0f, YarnConfiguration.DEFAULT_APPLICATION_TYPE, null); } diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestClientServiceDelegate.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestClientServiceDelegate.java index b85f18db4eb..275456df837 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestClientServiceDelegate.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestClientServiceDelegate.java @@ -515,7 +515,7 @@ private ApplicationReport getFinishedApplicationReport() { appId, 0); return ApplicationReport.newInstance(appId, attemptId, "user", "queue", "appname", "host", 124, null, YarnApplicationState.FINISHED, - "diagnostics", "url", 0, 0, FinalApplicationStatus.SUCCEEDED, null, + "diagnostics", "url", 0, 0, 0, FinalApplicationStatus.SUCCEEDED, null, "N/A", 0.0f, YarnConfiguration.DEFAULT_APPLICATION_TYPE, null); } @@ -525,7 +525,7 @@ private ApplicationReport getRunningApplicationReport(String host, int port) { appId, 0); return ApplicationReport.newInstance(appId, attemptId, "user", "queue", "appname", host, port, null, YarnApplicationState.RUNNING, "diagnostics", - "url", 0, 0, FinalApplicationStatus.UNDEFINED, null, "N/A", 0.0f, + "url", 0, 0, 0, FinalApplicationStatus.UNDEFINED, null, "N/A", 0.0f, YarnConfiguration.DEFAULT_APPLICATION_TYPE, null); } diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestYARNRunner.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestYARNRunner.java index 55ddea6da38..ff59f6a1a1b 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestYARNRunner.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestYARNRunner.java @@ -206,7 +206,7 @@ public ClientServiceDelegate answer(InvocationOnMock invocation) .thenReturn( ApplicationReport.newInstance(appId, null, "tmp", "tmp", "tmp", "tmp", 0, null, YarnApplicationState.FINISHED, "tmp", "tmp", - 0l, 0l, FinalApplicationStatus.SUCCEEDED, null, null, 0f, + 0l, 0l, 0l, FinalApplicationStatus.SUCCEEDED, null, null, 0f, "tmp", null)); yarnRunner.killJob(jobId); verify(clientDelegate).killJob(jobId); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ApplicationReport.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ApplicationReport.java index 9e9ec3cd2c9..ca895dfbb16 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ApplicationReport.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ApplicationReport.java @@ -58,7 +58,8 @@ public static ApplicationReport newInstance(ApplicationId applicationId, ApplicationAttemptId applicationAttemptId, String user, String queue, String name, String host, int rpcPort, Token clientToAMToken, YarnApplicationState state, String diagnostics, String url, - long startTime, long finishTime, FinalApplicationStatus finalStatus, + long startTime, long launchTime, long finishTime, + FinalApplicationStatus finalStatus, ApplicationResourceUsageReport appResources, String origTrackingUrl, float progress, String applicationType, Token amRmToken) { ApplicationReport report = Records.newRecord(ApplicationReport.class); @@ -74,6 +75,7 @@ public static ApplicationReport newInstance(ApplicationId applicationId, report.setDiagnostics(diagnostics); report.setTrackingUrl(url); report.setStartTime(startTime); + report.setLaunchTime(launchTime); report.setFinishTime(finishTime); report.setFinalApplicationStatus(finalStatus); report.setApplicationResourceUsageReport(appResources); @@ -90,16 +92,17 @@ public static ApplicationReport newInstance(ApplicationId applicationId, ApplicationAttemptId applicationAttemptId, String user, String queue, String name, String host, int rpcPort, Token clientToAMToken, YarnApplicationState state, String diagnostics, String url, - long startTime, long finishTime, FinalApplicationStatus finalStatus, + long startTime, long launchTime, long finishTime, + FinalApplicationStatus finalStatus, ApplicationResourceUsageReport appResources, String origTrackingUrl, float progress, String applicationType, Token amRmToken, Set tags, boolean unmanagedApplication, Priority priority, String appNodeLabelExpression, String amNodeLabelExpression) { ApplicationReport report = newInstance(applicationId, applicationAttemptId, user, queue, name, - host, rpcPort, clientToAMToken, state, diagnostics, url, startTime, - finishTime, finalStatus, appResources, origTrackingUrl, progress, - applicationType, amRmToken); + host, rpcPort, clientToAMToken, state, diagnostics, url, startTime, + launchTime, finishTime, finalStatus, appResources, origTrackingUrl, + progress, applicationType, amRmToken); report.setApplicationTags(tags); report.setUnmanagedApp(unmanagedApplication); report.setPriority(priority); @@ -283,6 +286,18 @@ public static ApplicationReport newInstance(ApplicationId applicationId, public abstract void setStartTime(long startTime); /** + * Get the launch time of the application. + * @return launch time of the application + */ + @Public + @Stable + public abstract long getLaunchTime(); + + @Private + @Unstable + public abstract void setLaunchTime(long launchTime); + + /** * Get the finish time of the application. * @return finish time of the application */ diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/server/application_history_server.proto b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/server/application_history_server.proto index 0fcf2ac61af..b7e30c63c0a 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/server/application_history_server.proto +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/server/application_history_server.proto @@ -32,10 +32,11 @@ message ApplicationHistoryDataProto { optional string queue = 5; optional int64 submit_time = 6; optional int64 start_time = 7; - optional int64 finish_time = 8; - optional string diagnostics_info = 9; - optional FinalApplicationStatusProto final_application_status = 10; - optional YarnApplicationStateProto yarn_application_state = 11; + optional int64 launch_time = 8; + optional int64 finish_time = 9; + optional string diagnostics_info = 10; + optional FinalApplicationStatusProto final_application_status = 11; + optional YarnApplicationStateProto yarn_application_state = 12; } message ApplicationStartDataProto { @@ -50,10 +51,11 @@ message ApplicationStartDataProto { message ApplicationFinishDataProto { optional ApplicationIdProto application_id = 1; - optional int64 finish_time = 2; - optional string diagnostics_info = 3; - optional FinalApplicationStatusProto final_application_status = 4; - optional YarnApplicationStateProto yarn_application_state = 5; + optional int64 launch_time = 2; + optional int64 finish_time = 3; + optional string diagnostics_info = 4; + optional FinalApplicationStatusProto final_application_status = 5; + optional YarnApplicationStateProto yarn_application_state = 6; } message ApplicationAttemptHistoryDataProto { diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/yarn_protos.proto b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/yarn_protos.proto index 81ebd798bb9..c7b2fb9941a 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/yarn_protos.proto +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/yarn_protos.proto @@ -200,21 +200,22 @@ message ApplicationReportProto { optional string trackingUrl = 9; optional string diagnostics = 10 [default = "N/A"]; optional int64 startTime = 11; - optional int64 finishTime = 12; - optional FinalApplicationStatusProto final_application_status = 13; - optional ApplicationResourceUsageReportProto app_resource_Usage = 14; - optional string originalTrackingUrl = 15; - optional ApplicationAttemptIdProto currentApplicationAttemptId = 16; - optional float progress = 17; - optional string applicationType = 18; - optional hadoop.common.TokenProto am_rm_token = 19; - repeated string applicationTags = 20; - optional LogAggregationStatusProto log_aggregation_status = 21; - optional bool unmanaged_application = 22 [default = false]; - optional PriorityProto priority = 23; - optional string appNodeLabelExpression = 24; - optional string amNodeLabelExpression = 25; - repeated AppTimeoutsMapProto appTimeouts = 26; + optional int64 launchTime = 12; + optional int64 finishTime = 13; + optional FinalApplicationStatusProto final_application_status = 14; + optional ApplicationResourceUsageReportProto app_resource_Usage = 15; + optional string originalTrackingUrl = 16; + optional ApplicationAttemptIdProto currentApplicationAttemptId = 17; + optional float progress = 18; + optional string applicationType = 19; + optional hadoop.common.TokenProto am_rm_token = 20; + repeated string applicationTags = 21; + optional LogAggregationStatusProto log_aggregation_status = 22; + optional bool unmanaged_application = 23 [default = false]; + optional PriorityProto priority = 24; + optional string appNodeLabelExpression = 25; + optional string amNodeLabelExpression = 26; + repeated AppTimeoutsMapProto appTimeouts = 27; } message AppTimeoutsMapProto { diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/ProtocolHATestBase.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/ProtocolHATestBase.java index a8e91323a38..72e6979917c 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/ProtocolHATestBase.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/ProtocolHATestBase.java @@ -625,7 +625,7 @@ public ApplicationReport createFakeAppReport() { ApplicationReport.newInstance(appId, attemptId, "fakeUser", "fakeQueue", "fakeApplicationName", "localhost", 0, null, YarnApplicationState.FINISHED, "fake an application report", "", - 1000l, 1200l, FinalApplicationStatus.FAILED, null, "", 50f, + 1000l, 1000l, 1200l, FinalApplicationStatus.FAILED, null, "", 50f, "fakeApplicationType", null); return report; } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/api/impl/TestAHSClient.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/api/impl/TestAHSClient.java index c3e3c414771..f0e3ca2a4cc 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/api/impl/TestAHSClient.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/api/impl/TestAHSClient.java @@ -332,7 +332,7 @@ private void createAppReports() { ApplicationReport.newInstance(applicationId, ApplicationAttemptId.newInstance(applicationId, 1), "user", "queue", "appname", "host", 124, null, - YarnApplicationState.RUNNING, "diagnostics", "url", 0, 0, + YarnApplicationState.RUNNING, "diagnostics", "url", 0, 0, 0, FinalApplicationStatus.SUCCEEDED, null, "N/A", 0.53789f, "YARN", null); List applicationReports = @@ -389,7 +389,7 @@ private void createAppReports() { ApplicationReport.newInstance(applicationId2, ApplicationAttemptId.newInstance(applicationId2, 2), "user2", "queue2", "appname2", "host2", 125, null, - YarnApplicationState.FINISHED, "diagnostics2", "url2", 2, 2, + YarnApplicationState.FINISHED, "diagnostics2", "url2", 2, 2, 2, FinalApplicationStatus.SUCCEEDED, null, "N/A", 0.63789f, "NON-YARN", null); applicationReports.add(newApplicationReport2); @@ -399,7 +399,7 @@ private void createAppReports() { ApplicationReport.newInstance(applicationId3, ApplicationAttemptId.newInstance(applicationId3, 3), "user3", "queue3", "appname3", "host3", 126, null, - YarnApplicationState.RUNNING, "diagnostics3", "url3", 3, 3, + YarnApplicationState.RUNNING, "diagnostics3", "url3", 3, 3, 3, FinalApplicationStatus.SUCCEEDED, null, "N/A", 0.73789f, "MAPREDUCE", null); applicationReports.add(newApplicationReport3); @@ -409,7 +409,7 @@ private void createAppReports() { ApplicationReport.newInstance(applicationId4, ApplicationAttemptId.newInstance(applicationId4, 4), "user4", "queue4", "appname4", "host4", 127, null, - YarnApplicationState.FAILED, "diagnostics4", "url4", 4, 4, + YarnApplicationState.FAILED, "diagnostics4", "url4", 4, 4, 4, FinalApplicationStatus.SUCCEEDED, null, "N/A", 0.83789f, "NON-MAPREDUCE", null); applicationReports.add(newApplicationReport4); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/api/impl/TestYarnClient.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/api/impl/TestYarnClient.java index 41ef404be6b..813061e2af7 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/api/impl/TestYarnClient.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/api/impl/TestYarnClient.java @@ -752,7 +752,7 @@ public void setYarnApplicationState(YarnApplicationState state) { ApplicationReport newApplicationReport = ApplicationReport.newInstance( applicationId, ApplicationAttemptId.newInstance(applicationId, 1), "user", "queue", "appname", "host", 124, null, - YarnApplicationState.RUNNING, "diagnostics", "url", 0, 0, + YarnApplicationState.RUNNING, "diagnostics", "url", 0, 0, 0, FinalApplicationStatus.SUCCEEDED, null, "N/A", 0.53789f, "YARN", null); List applicationReports = new ArrayList(); applicationReports.add(newApplicationReport); @@ -827,7 +827,7 @@ public void setYarnApplicationState(YarnApplicationState state) { ApplicationReport newApplicationReport2 = ApplicationReport.newInstance( applicationId2, ApplicationAttemptId.newInstance(applicationId2, 2), "user2", "queue2", "appname2", "host2", 125, null, - YarnApplicationState.FINISHED, "diagnostics2", "url2", 2, 2, + YarnApplicationState.FINISHED, "diagnostics2", "url2", 2, 2, 2, FinalApplicationStatus.SUCCEEDED, null, "N/A", 0.63789f, "NON-YARN", null); applicationReports.add(newApplicationReport2); @@ -836,7 +836,7 @@ public void setYarnApplicationState(YarnApplicationState state) { ApplicationReport newApplicationReport3 = ApplicationReport.newInstance( applicationId3, ApplicationAttemptId.newInstance(applicationId3, 3), "user3", "queue3", "appname3", "host3", 126, null, - YarnApplicationState.RUNNING, "diagnostics3", "url3", 3, 3, + YarnApplicationState.RUNNING, "diagnostics3", "url3", 3, 3, 3, FinalApplicationStatus.SUCCEEDED, null, "N/A", 0.73789f, "MAPREDUCE", null); applicationReports.add(newApplicationReport3); @@ -847,7 +847,7 @@ public void setYarnApplicationState(YarnApplicationState state) { applicationId4, ApplicationAttemptId.newInstance(applicationId4, 4), "user4", "queue4", "appname4", "host4", 127, null, - YarnApplicationState.FAILED, "diagnostics4", "url4", 4, 4, + YarnApplicationState.FAILED, "diagnostics4", "url4", 4, 4, 4, FinalApplicationStatus.SUCCEEDED, null, "N/A", 0.83789f, "NON-MAPREDUCE", null); applicationReports.add(newApplicationReport4); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/cli/TestYarnCLI.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/cli/TestYarnCLI.java index 3c35b9cd313..4989c5a1a5b 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/cli/TestYarnCLI.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/cli/TestYarnCLI.java @@ -123,7 +123,7 @@ public void testGetApplicationReport() throws Exception { ApplicationReport newApplicationReport = ApplicationReport.newInstance( applicationId, ApplicationAttemptId.newInstance(applicationId, 1), "user", "queue", "appname", "host", 124, null, - YarnApplicationState.FINISHED, "diagnostics", "url", 0, 0, + YarnApplicationState.FINISHED, "diagnostics", "url", 0, 0, 0, FinalApplicationStatus.SUCCEEDED, usageReport, "N/A", 0.53789f, "YARN", null, null, false, Priority.newInstance(0), "high-mem", "high-mem"); newApplicationReport.setLogAggregationStatus(LogAggregationStatus.SUCCEEDED); @@ -370,7 +370,7 @@ public void testGetApplications() throws Exception { ApplicationReport newApplicationReport = ApplicationReport.newInstance( applicationId, ApplicationAttemptId.newInstance(applicationId, 1), "user", "queue", "appname", "host", 124, null, - YarnApplicationState.RUNNING, "diagnostics", "url", 0, 0, + YarnApplicationState.RUNNING, "diagnostics", "url", 0, 0, 0, FinalApplicationStatus.SUCCEEDED, null, "N/A", 0.53789f, "YARN", null, Sets.newHashSet("tag1", "tag3"), false, Priority.UNDEFINED, "", ""); List applicationReports = @@ -381,7 +381,7 @@ public void testGetApplications() throws Exception { ApplicationReport newApplicationReport2 = ApplicationReport.newInstance( applicationId2, ApplicationAttemptId.newInstance(applicationId2, 2), "user2", "queue2", "appname2", "host2", 125, null, - YarnApplicationState.FINISHED, "diagnostics2", "url2", 2, 2, + YarnApplicationState.FINISHED, "diagnostics2", "url2", 2, 2, 2, FinalApplicationStatus.SUCCEEDED, null, "N/A", 0.63789f, "NON-YARN", null, Sets.newHashSet("tag2", "tag3"), false, Priority.UNDEFINED, "", ""); @@ -391,7 +391,7 @@ public void testGetApplications() throws Exception { ApplicationReport newApplicationReport3 = ApplicationReport.newInstance( applicationId3, ApplicationAttemptId.newInstance(applicationId3, 3), "user3", "queue3", "appname3", "host3", 126, null, - YarnApplicationState.RUNNING, "diagnostics3", "url3", 3, 3, + YarnApplicationState.RUNNING, "diagnostics3", "url3", 3, 3, 3, FinalApplicationStatus.SUCCEEDED, null, "N/A", 0.73789f, "MAPREDUCE", null, Sets.newHashSet("tag1", "tag4"), false, Priority.UNDEFINED, "", ""); @@ -401,7 +401,7 @@ public void testGetApplications() throws Exception { ApplicationReport newApplicationReport4 = ApplicationReport.newInstance( applicationId4, ApplicationAttemptId.newInstance(applicationId4, 4), "user4", "queue4", "appname4", "host4", 127, null, - YarnApplicationState.FAILED, "diagnostics4", "url4", 4, 4, + YarnApplicationState.FAILED, "diagnostics4", "url4", 4, 4, 4, FinalApplicationStatus.SUCCEEDED, null, "N/A", 0.83789f, "NON-MAPREDUCE", null, Sets.newHashSet("tag1"), false, Priority.UNDEFINED, "", ""); @@ -411,7 +411,7 @@ public void testGetApplications() throws Exception { ApplicationReport newApplicationReport5 = ApplicationReport.newInstance( applicationId5, ApplicationAttemptId.newInstance(applicationId5, 5), "user5", "queue5", "appname5", "host5", 128, null, - YarnApplicationState.ACCEPTED, "diagnostics5", "url5", 5, 5, + YarnApplicationState.ACCEPTED, "diagnostics5", "url5", 5, 5, 5, FinalApplicationStatus.KILLED, null, "N/A", 0.93789f, "HIVE", null, Sets.newHashSet("tag2", "tag4"), false, Priority.UNDEFINED, "", ""); applicationReports.add(newApplicationReport5); @@ -420,7 +420,7 @@ public void testGetApplications() throws Exception { ApplicationReport newApplicationReport6 = ApplicationReport.newInstance( applicationId6, ApplicationAttemptId.newInstance(applicationId6, 6), "user6", "queue6", "appname6", "host6", 129, null, - YarnApplicationState.SUBMITTED, "diagnostics6", "url6", 6, 6, + YarnApplicationState.SUBMITTED, "diagnostics6", "url6", 6, 6, 6, FinalApplicationStatus.KILLED, null, "N/A", 0.99789f, "PIG", null, new HashSet(), false, Priority.UNDEFINED, "", ""); applicationReports.add(newApplicationReport6); @@ -994,7 +994,7 @@ public void testKillApplication() throws Exception { ApplicationReport newApplicationReport2 = ApplicationReport.newInstance( applicationId, ApplicationAttemptId.newInstance(applicationId, 1), "user", "queue", "appname", "host", 124, null, - YarnApplicationState.FINISHED, "diagnostics", "url", 0, 0, + YarnApplicationState.FINISHED, "diagnostics", "url", 0, 0, 0, FinalApplicationStatus.SUCCEEDED, null, "N/A", 0.53789f, "YARN", null); when(client.getApplicationReport(any(ApplicationId.class))).thenReturn( newApplicationReport2); @@ -1007,7 +1007,7 @@ public void testKillApplication() throws Exception { ApplicationReport newApplicationReport = ApplicationReport.newInstance( applicationId, ApplicationAttemptId.newInstance(applicationId, 1), "user", "queue", "appname", "host", 124, null, - YarnApplicationState.RUNNING, "diagnostics", "url", 0, 0, + YarnApplicationState.RUNNING, "diagnostics", "url", 0, 0, 0, FinalApplicationStatus.SUCCEEDED, null, "N/A", 0.53789f, "YARN", null); when(client.getApplicationReport(any(ApplicationId.class))).thenReturn( newApplicationReport); @@ -1046,12 +1046,12 @@ public void testKillApplications() throws Exception { ApplicationReport newApplicationReport1 = ApplicationReport.newInstance( applicationId1, ApplicationAttemptId.newInstance(applicationId1, 1), "user", "queue", "appname", "host", 124, null, - YarnApplicationState.FINISHED, "diagnostics", "url", 0, 0, + YarnApplicationState.FINISHED, "diagnostics", "url", 0, 0, 0, FinalApplicationStatus.SUCCEEDED, null, "N/A", 0.53789f, "YARN", null); ApplicationReport newApplicationReport2 = ApplicationReport.newInstance( applicationId2, ApplicationAttemptId.newInstance(applicationId2, 1), "user", "queue", "appname", "host", 124, null, - YarnApplicationState.FINISHED, "diagnostics", "url", 0, 0, + YarnApplicationState.FINISHED, "diagnostics", "url", 0, 0, 0, FinalApplicationStatus.SUCCEEDED, null, "N/A", 0.34344f, "YARN", null); when(client.getApplicationReport(applicationId1)).thenReturn( newApplicationReport1); @@ -1071,12 +1071,12 @@ public void testKillApplications() throws Exception { ApplicationReport newApplicationReport3 = ApplicationReport.newInstance( applicationId1, ApplicationAttemptId.newInstance(applicationId1, 1), "user", "queue", "appname", "host", 124, null, - YarnApplicationState.RUNNING, "diagnostics", "url", 0, 0, + YarnApplicationState.RUNNING, "diagnostics", "url", 0, 0, 0, FinalApplicationStatus.SUCCEEDED, null, "N/A", 0.53789f, "YARN", null); ApplicationReport newApplicationReport4 = ApplicationReport.newInstance( applicationId2, ApplicationAttemptId.newInstance(applicationId2, 1), "user", "queue", "appname", "host", 124, null, - YarnApplicationState.RUNNING, "diagnostics", "url", 0, 0, + YarnApplicationState.RUNNING, "diagnostics", "url", 0, 0, 0, FinalApplicationStatus.SUCCEEDED, null, "N/A", 0.53345f, "YARN", null); when(client.getApplicationReport(applicationId1)).thenReturn( newApplicationReport3); @@ -1114,7 +1114,7 @@ public void testKillApplications() throws Exception { ApplicationReport newApplicationReport5 = ApplicationReport.newInstance( applicationId1, ApplicationAttemptId.newInstance(applicationId1, 1), "user", "queue", "appname", "host", 124, null, - YarnApplicationState.RUNNING, "diagnostics", "url", 0, 0, + YarnApplicationState.RUNNING, "diagnostics", "url", 0, 0, 0, FinalApplicationStatus.SUCCEEDED, null, "N/A", 0.53345f, "YARN", null); when(client.getApplicationReport(applicationId1)).thenReturn( newApplicationReport5); @@ -1141,12 +1141,12 @@ public void testKillApplicationsOfDifferentEndStates() throws Exception { ApplicationReport newApplicationReport5 = ApplicationReport.newInstance( applicationId1, ApplicationAttemptId.newInstance(applicationId1, 1), "user", "queue", "appname", "host", 124, null, - YarnApplicationState.FINISHED, "diagnostics", "url", 0, 0, + YarnApplicationState.FINISHED, "diagnostics", "url", 0, 0, 0, FinalApplicationStatus.SUCCEEDED, null, "N/A", 0.53789f, "YARN", null); ApplicationReport newApplicationReport6 = ApplicationReport.newInstance( applicationId2, ApplicationAttemptId.newInstance(applicationId2, 1), "user", "queue", "appname", "host", 124, null, - YarnApplicationState.RUNNING, "diagnostics", "url", 0, 0, + YarnApplicationState.RUNNING, "diagnostics", "url", 0, 0, 0, FinalApplicationStatus.SUCCEEDED, null, "N/A", 0.53345f, "YARN", null); when(client.getApplicationReport(applicationId1)).thenReturn( newApplicationReport5); @@ -1169,7 +1169,7 @@ public void testMoveApplicationAcrossQueues() throws Exception { ApplicationReport newApplicationReport2 = ApplicationReport.newInstance( applicationId, ApplicationAttemptId.newInstance(applicationId, 1), "user", "queue", "appname", "host", 124, null, - YarnApplicationState.FINISHED, "diagnostics", "url", 0, 0, + YarnApplicationState.FINISHED, "diagnostics", "url", 0, 0, 0, FinalApplicationStatus.SUCCEEDED, null, "N/A", 0.53789f, "YARN", null); when(client.getApplicationReport(any(ApplicationId.class))).thenReturn( newApplicationReport2); @@ -1184,7 +1184,7 @@ public void testMoveApplicationAcrossQueues() throws Exception { ApplicationReport newApplicationReport = ApplicationReport.newInstance( applicationId, ApplicationAttemptId.newInstance(applicationId, 1), "user", "queue", "appname", "host", 124, null, - YarnApplicationState.RUNNING, "diagnostics", "url", 0, 0, + YarnApplicationState.RUNNING, "diagnostics", "url", 0, 0, 0, FinalApplicationStatus.SUCCEEDED, null, "N/A", 0.53789f, "YARN", null); when(client.getApplicationReport(any(ApplicationId.class))).thenReturn( newApplicationReport); @@ -1219,7 +1219,7 @@ public void testMoveApplicationAcrossQueuesWithNewCommand() throws Exception { ApplicationReport newApplicationReport2 = ApplicationReport.newInstance( applicationId, ApplicationAttemptId.newInstance(applicationId, 1), "user", "queue", "appname", "host", 124, null, - YarnApplicationState.FINISHED, "diagnostics", "url", 0, 0, + YarnApplicationState.FINISHED, "diagnostics", "url", 0, 0, 0, FinalApplicationStatus.SUCCEEDED, null, "N/A", 0.53789f, "YARN", null); when(client.getApplicationReport(any(ApplicationId.class))) .thenReturn(newApplicationReport2); @@ -1234,7 +1234,7 @@ public void testMoveApplicationAcrossQueuesWithNewCommand() throws Exception { ApplicationReport newApplicationReport = ApplicationReport.newInstance( applicationId, ApplicationAttemptId.newInstance(applicationId, 1), "user", "queue", "appname", "host", 124, null, - YarnApplicationState.RUNNING, "diagnostics", "url", 0, 0, + YarnApplicationState.RUNNING, "diagnostics", "url", 0, 0, 0, FinalApplicationStatus.SUCCEEDED, null, "N/A", 0.53789f, "YARN", null); when(client.getApplicationReport(any(ApplicationId.class))) .thenReturn(newApplicationReport); @@ -1922,7 +1922,7 @@ public void testUpdateApplicationPriority() throws Exception { ApplicationReport.newInstance(applicationId, ApplicationAttemptId.newInstance(applicationId, 1), "user", "queue", "appname", "host", 124, null, - YarnApplicationState.RUNNING, "diagnostics", "url", 0, 0, + YarnApplicationState.RUNNING, "diagnostics", "url", 0, 0, 0, FinalApplicationStatus.UNDEFINED, null, "N/A", 0.53789f, "YARN", null); when(client.getApplicationReport(any(ApplicationId.class))).thenReturn( @@ -2151,7 +2151,7 @@ public void testUpdateApplicationTimeout() throws Exception { ApplicationReport appReport = ApplicationReport.newInstance(applicationId, ApplicationAttemptId.newInstance(applicationId, 1), "user", "queue", "appname", "host", 124, null, YarnApplicationState.RUNNING, - "diagnostics", "url", 0, 0, FinalApplicationStatus.UNDEFINED, null, + "diagnostics", "url", 0, 0, 0, FinalApplicationStatus.UNDEFINED, null, "N/A", 0.53789f, "YARN", null); ApplicationTimeout timeout = ApplicationTimeout .newInstance(ApplicationTimeoutType.LIFETIME, "N/A", -1); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ApplicationReportPBImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ApplicationReportPBImpl.java index 036fa907fe6..cfb959d0fac 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ApplicationReportPBImpl.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ApplicationReportPBImpl.java @@ -221,6 +221,12 @@ public long getStartTime() { } @Override + public long getLaunchTime() { + ApplicationReportProtoOrBuilder p = viaProto ? proto : builder; + return p.getLaunchTime(); + } + + @Override public long getFinishTime() { ApplicationReportProtoOrBuilder p = viaProto ? proto : builder; return p.getFinishTime(); @@ -414,6 +420,12 @@ public void setStartTime(long startTime) { } @Override + public void setLaunchTime(long launchTime) { + maybeInitBuilder(); + builder.setLaunchTime(launchTime); + } + + @Override public void setFinishTime(long finishTime) { maybeInitBuilder(); builder.setFinishTime(finishTime); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/api/TestApplicatonReport.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/api/TestApplicatonReport.java index 46fc4d58d92..f0d41b7b2ff 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/api/TestApplicatonReport.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/api/TestApplicatonReport.java @@ -58,11 +58,10 @@ protected static ApplicationReport createApplicationReport( ApplicationReport appReport = ApplicationReport.newInstance(appId, appAttemptId, "user", "queue", "appname", "host", 124, null, YarnApplicationState.FINISHED, - "diagnostics", "url", 0, 0, FinalApplicationStatus.SUCCEEDED, null, + "diagnostics", "url", 0, 0, 0, FinalApplicationStatus.SUCCEEDED, null, "N/A", 0.53789f, YarnConfiguration.DEFAULT_APPLICATION_TYPE, null, null, false, Priority.newInstance(0),"",""); return appReport; } } - diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/ApplicationHistoryManagerImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/ApplicationHistoryManagerImpl.java index b8931d81abc..473985479ed 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/ApplicationHistoryManagerImpl.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/ApplicationHistoryManagerImpl.java @@ -137,7 +137,8 @@ private ApplicationReport convertToApplicationReport( currentApplicationAttemptId, appHistory.getUser(), appHistory.getQueue(), appHistory.getApplicationName(), host, rpcPort, null, appHistory.getYarnApplicationState(), appHistory.getDiagnosticsInfo(), - trackingUrl, appHistory.getStartTime(), appHistory.getFinishTime(), + trackingUrl, appHistory.getStartTime(), + appHistory.getLaunchTime(), appHistory.getFinishTime(), appHistory.getFinalApplicationStatus(), null, "", 100, appHistory.getApplicationType(), null); } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/ApplicationHistoryManagerOnTimelineStore.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/ApplicationHistoryManagerOnTimelineStore.java index 9240ed872e0..7a540519c33 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/ApplicationHistoryManagerOnTimelineStore.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/ApplicationHistoryManagerOnTimelineStore.java @@ -249,6 +249,7 @@ private static ApplicationReportExt convertToApplicationReport( String type = null; boolean unmanagedApplication = false; long createdTime = 0; + long launchedTime = 0; long finishedTime = 0; float progress = 0.0f; int applicationPriority = 0; @@ -280,7 +281,7 @@ private static ApplicationReportExt convertToApplicationReport( return new ApplicationReportExt(ApplicationReport.newInstance( ApplicationId.fromString(entity.getEntityId()), latestApplicationAttemptId, user, queue, name, null, -1, null, - state, diagnosticsInfo, null, createdTime, finishedTime, + state, diagnosticsInfo, null, createdTime, launchedTime, finishedTime, finalStatus, null, null, progress, type, null, appTags, unmanagedApplication, Priority.newInstance(applicationPriority), appNodeLabelExpression, amNodeLabelExpression), appViewACLs); @@ -438,7 +439,7 @@ private static ApplicationReportExt convertToApplicationReport( return new ApplicationReportExt(ApplicationReport.newInstance( ApplicationId.fromString(entity.getEntityId()), latestApplicationAttemptId, user, queue, name, null, -1, null, state, - diagnosticsInfo, null, createdTime, finishedTime, finalStatus, + diagnosticsInfo, null, createdTime, launchedTime, finishedTime, finalStatus, appResources, null, progress, type, null, appTags, unmanagedApplication, Priority.newInstance(applicationPriority), appNodeLabelExpression, amNodeLabelExpression), appViewACLs); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/FileSystemApplicationHistoryStore.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/FileSystemApplicationHistoryStore.java index fa2da44f5f2..2b56009d856 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/FileSystemApplicationHistoryStore.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/FileSystemApplicationHistoryStore.java @@ -155,7 +155,7 @@ public ApplicationHistoryData getApplication(ApplicationId appId) boolean readFinishData = false; ApplicationHistoryData historyData = ApplicationHistoryData.newInstance(appId, null, null, null, null, - Long.MIN_VALUE, Long.MIN_VALUE, Long.MAX_VALUE, null, + Long.MIN_VALUE, Long.MIN_VALUE, Long.MAX_VALUE, Long.MAX_VALUE, null, FinalApplicationStatus.UNDEFINED, null); while ((!readStartData || !readFinishData) && hfReader.hasNext()) { HistoryFileReader.Entry entry = hfReader.next(); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/MemoryApplicationHistoryStore.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/MemoryApplicationHistoryStore.java index c226ad3c90e..9324f6b3bf3 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/MemoryApplicationHistoryStore.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/MemoryApplicationHistoryStore.java @@ -142,7 +142,7 @@ public void applicationStarted(ApplicationStartData appStart) ApplicationHistoryData.newInstance(appStart.getApplicationId(), appStart.getApplicationName(), appStart.getApplicationType(), appStart.getQueue(), appStart.getUser(), appStart.getSubmitTime(), - appStart.getStartTime(), Long.MAX_VALUE, null, null, null)); + appStart.getStartTime(), Long.MAX_VALUE, Long.MAX_VALUE, null, null, null)); if (oldData != null) { throw new IOException("The start information of application " + appStart.getApplicationId() + " is already stored."); @@ -165,6 +165,7 @@ public void applicationFinished(ApplicationFinishData appFinish) throw new IOException("The finish information of application " + appFinish.getApplicationId() + " is already stored."); } + data.setLaunchTime(appFinish.getLaunchTime()); data.setFinishTime(appFinish.getFinishTime()); data.setDiagnosticsInfo(appFinish.getDiagnosticsInfo()); data.setFinalApplicationStatus(appFinish.getFinalApplicationStatus()); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/records/ApplicationFinishData.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/records/ApplicationFinishData.java index 997fa6cbfd8..81316e5d2f7 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/records/ApplicationFinishData.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/records/ApplicationFinishData.java @@ -36,12 +36,13 @@ @Public @Unstable public static ApplicationFinishData newInstance(ApplicationId applicationId, - long finishTime, String diagnosticsInfo, + long launchTime, long finishTime, String diagnosticsInfo, FinalApplicationStatus finalApplicationStatus, YarnApplicationState yarnApplicationState) { ApplicationFinishData appFD = Records.newRecord(ApplicationFinishData.class); appFD.setApplicationId(applicationId); + appFD.setLaunchTime(launchTime); appFD.setFinishTime(finishTime); appFD.setDiagnosticsInfo(diagnosticsInfo); appFD.setFinalApplicationStatus(finalApplicationStatus); @@ -59,6 +60,14 @@ public static ApplicationFinishData newInstance(ApplicationId applicationId, @Public @Unstable + public abstract long getLaunchTime(); + + @Public + @Unstable + public abstract void setLaunchTime(long finishTime); + + @Public + @Unstable public abstract long getFinishTime(); @Public diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/records/ApplicationHistoryData.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/records/ApplicationHistoryData.java index b7d16f3c412..6333485bd3f 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/records/ApplicationHistoryData.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/records/ApplicationHistoryData.java @@ -46,6 +46,8 @@ private long startTime; + private long launchTime; + private long finishTime; private String diagnosticsInfo; @@ -58,7 +60,7 @@ @Unstable public static ApplicationHistoryData newInstance(ApplicationId applicationId, String applicationName, String applicationType, String queue, - String user, long submitTime, long startTime, long finishTime, + String user, long submitTime, long startTime, long launchTime, long finishTime, String diagnosticsInfo, FinalApplicationStatus finalApplicationStatus, YarnApplicationState yarnApplicationState) { ApplicationHistoryData appHD = new ApplicationHistoryData(); @@ -69,6 +71,7 @@ public static ApplicationHistoryData newInstance(ApplicationId applicationId, appHD.setUser(user); appHD.setSubmitTime(submitTime); appHD.setStartTime(startTime); + appHD.setLaunchTime(launchTime); appHD.setFinishTime(finishTime); appHD.setDiagnosticsInfo(diagnosticsInfo); appHD.setFinalApplicationStatus(finalApplicationStatus); @@ -162,6 +165,18 @@ public void setStartTime(long startTime) { @Public @Unstable + public long getLaunchTime() { + return launchTime; + } + + @Public + @Unstable + public void setLaunchTime(long launchTime) { + this.launchTime = launchTime; + } + + @Public + @Unstable public long getFinishTime() { return finishTime; } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/records/impl/pb/ApplicationFinishDataPBImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/records/impl/pb/ApplicationFinishDataPBImpl.java index 337426d0284..aad78b1b05c 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/records/impl/pb/ApplicationFinishDataPBImpl.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/records/impl/pb/ApplicationFinishDataPBImpl.java @@ -73,6 +73,18 @@ public void setApplicationId(ApplicationId applicationId) { } @Override + public long getLaunchTime() { + ApplicationFinishDataProtoOrBuilder p = viaProto ? proto : builder; + return p.getLaunchTime(); + } + + @Override + public void setLaunchTime(long launchTime) { + maybeInitBuilder(); + builder.setLaunchTime(launchTime); + } + + @Override public long getFinishTime() { ApplicationFinishDataProtoOrBuilder p = viaProto ? proto : builder; return p.getFinishTime(); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/ApplicationHistoryStoreTestUtils.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/ApplicationHistoryStoreTestUtils.java index de4051a494c..98f5b65d08f 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/ApplicationHistoryStoreTestUtils.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/ApplicationHistoryStoreTestUtils.java @@ -49,7 +49,7 @@ protected void writeApplicationStartData(ApplicationId appId) protected void writeApplicationFinishData(ApplicationId appId) throws IOException { - store.applicationFinished(ApplicationFinishData.newInstance(appId, 0, + store.applicationFinished(ApplicationFinishData.newInstance(appId, 0, 0, appId.toString(), FinalApplicationStatus.UNDEFINED, YarnApplicationState.FINISHED)); } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/utils/BuilderUtils.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/utils/BuilderUtils.java index e7f47af2647..708bfcfccd4 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/utils/BuilderUtils.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/utils/BuilderUtils.java @@ -387,7 +387,7 @@ public static ApplicationReport newApplicationReport( ApplicationId applicationId, ApplicationAttemptId applicationAttemptId, String user, String queue, String name, String host, int rpcPort, Token clientToAMToken, YarnApplicationState state, String diagnostics, - String url, long startTime, long finishTime, + String url, long startTime, long launchTime, long finishTime, FinalApplicationStatus finalStatus, ApplicationResourceUsageReport appResources, String origTrackingUrl, float progress, String appType, Token amRmToken, Set tags, @@ -406,6 +406,7 @@ public static ApplicationReport newApplicationReport( report.setDiagnostics(diagnostics); report.setTrackingUrl(url); report.setStartTime(startTime); + report.setLaunchTime(launchTime); report.setFinishTime(finishTime); report.setFinalApplicationStatus(finalStatus); report.setApplicationResourceUsageReport(appResources); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/webapp/AppBlock.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/webapp/AppBlock.java index 693aa046df5..affe7c1fed2 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/webapp/AppBlock.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/webapp/AppBlock.java @@ -232,11 +232,11 @@ private void generateOverviewTable(AppInfo app, String schedulerPath, .__("Queue:", schedulerPath, app.getQueue()) .__("FinalStatus Reported by AM:", clairfyAppFinalStatus(app.getFinalAppStatus())) - .__("Started:", Times.format(app.getStartedTime())) - .__( - "Elapsed:", - StringUtils.formatTime(Times.elapsed(app.getStartedTime(), - app.getFinishedTime()))) + .__("Submitted:", Times.format(app.getStartedTime())) + .__("Pending:", StringUtils.formatTime(app.getPendingTime())) + .__("Started:", Times.format(app.getLaunchedTime())) + .__("Elapsed:", StringUtils.formatTime(app.getElapsedTime())) + .__("Finished:", Times.format(app.getFinishedTime())) .__( "Tracking URL:", app.getTrackingUrl() == null diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/webapp/AppsBlock.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/webapp/AppsBlock.java index d836e641177..30c87b960d2 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/webapp/AppsBlock.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/webapp/AppsBlock.java @@ -145,7 +145,8 @@ protected void renderData(Block html) { html.table("#apps").thead().tr().th(".id", "ID").th(".user", "User") .th(".name", "Name").th(".type", "Application Type") .th(".queue", "Queue").th(".priority", "Application Priority") - .th(".starttime", "StartTime").th(".finishtime", "FinishTime") + .th(".submitTime", "SubmitTime").th(".starttime", "StartTime") + .th(".finishtime", "FinishTime") .th(".state", "State").th(".finalstatus", "FinalStatus") .th(".progress", "Progress").th(".ui", "Tracking UI").__().__().tbody(); @@ -183,6 +184,7 @@ protected void renderData(Block html) { .getQueue()))).append("\",\"").append(String .valueOf(app.getPriority())) .append("\",\"").append(app.getStartedTime()) + .append("\",\"").append(app.getLaunchedTime()) .append("\",\"").append(app.getFinishedTime()) .append("\",\"") .append(app.getAppState() == null ? UNAVAILABLE : app.getAppState()) diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/webapp/WebPageUtils.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/webapp/WebPageUtils.java index 3a26ae58890..7f2d71401b7 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/webapp/WebPageUtils.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/webapp/WebPageUtils.java @@ -51,7 +51,7 @@ private static String getAppsTableColumnDefs( sb.append("[\n") .append("{'sType':'natural', 'aTargets': [0]") .append(", 'mRender': parseHadoopID }") - .append("\n, {'sType':'numeric', 'aTargets': [6, 7]") + .append("\n, {'sType':'numeric', 'aTargets': [6, 7, 8]") .append(", 'mRender': renderHadoopDate }") .append("\n, {'sType':'numeric', bSearchable:false, 'aTargets':"); if (isFairSchedulerPage) { diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/webapp/dao/AppInfo.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/webapp/dao/AppInfo.java index ac2f8da6aa2..b2d78b9c97f 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/webapp/dao/AppInfo.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/webapp/dao/AppInfo.java @@ -55,7 +55,9 @@ protected FinalApplicationStatus finalAppStatus; protected long submittedTime; protected long startedTime; + protected long launchedTime; protected long finishedTime; + protected long pendingTime; protected long elapsedTime; protected String applicationTags; protected int priority; @@ -88,8 +90,10 @@ public AppInfo(ApplicationReport app) { originalTrackingUrl = app.getOriginalTrackingUrl(); submittedTime = app.getStartTime(); startedTime = app.getStartTime(); + launchedTime = app.getLaunchTime(); finishedTime = app.getFinishTime(); - elapsedTime = Times.elapsed(startedTime, finishedTime); + pendingTime = Times.elapsed(startedTime, launchedTime); + elapsedTime = Times.elapsed(launchedTime, finishedTime); finalAppStatus = app.getFinalApplicationStatus(); priority = 0; if (app.getPriority() != null) { @@ -202,10 +206,18 @@ public long getStartedTime() { return startedTime; } + public long getLaunchedTime() { + return launchedTime; + } + public long getFinishedTime() { return finishedTime; } + public long getPendingTime() { + return pendingTime; + } + public long getElapsedTime() { return elapsedTime; } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ahs/RMApplicationHistoryWriter.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ahs/RMApplicationHistoryWriter.java index 19c5c27f961..7339f5f9be5 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ahs/RMApplicationHistoryWriter.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ahs/RMApplicationHistoryWriter.java @@ -238,8 +238,8 @@ public void applicationFinished(RMApp app, RMAppState finalState) { dispatcher.getEventHandler().handle( new WritingApplicationFinishEvent(app.getApplicationId(), ApplicationFinishData.newInstance(app.getApplicationId(), - app.getFinishTime(), app.getDiagnostics().toString(), - app.getFinalApplicationStatus(), + app.getLaunchTime(), app.getFinishTime(), + app.getDiagnostics().toString(), app.getFinalApplicationStatus(), RMServerUtils.createApplicationState(finalState)))); } } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/RMApp.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/RMApp.java index 43fd1fbfbdf..022ff94efd3 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/RMApp.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/RMApp.java @@ -172,7 +172,15 @@ ApplicationReport createAndGetApplicationReport(String clientUserName, * @return the submit time of the application. */ long getSubmitTime(); - + + /** + * the actual start time of the application + * since getStartTime() returns submit time + * this new field is to prevent potential backwards compatibility issues + * @return the actual start time of the application. + */ + long getLaunchTime(); + /** * The tracking url for the application master. * @return the tracking url for the application master. diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/RMAppImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/RMAppImpl.java index 03be7937b3a..a46aa061b86 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/RMAppImpl.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/RMAppImpl.java @@ -161,6 +161,7 @@ // Mutable fields private long startTime; + private long launchTime = 0; private long finishTime = 0; private long storedFinishTime = 0; private int firstAttemptIdInStateStore = 1; @@ -761,9 +762,9 @@ public ApplicationReport createAndGetApplicationReport(String clientUserName, this.applicationId, currentApplicationAttemptId, this.user, this.queue, this.name, host, rpcPort, clientToAMToken, createApplicationState(), diags, trackingUrl, this.startTime, - this.finishTime, finishState, appUsageReport, origTrackingUrl, - progress, this.applicationType, amrmToken, applicationTags, - this.getApplicationPriority()); + this.launchTime, this.finishTime, finishState, appUsageReport, + origTrackingUrl, progress, this.applicationType, amrmToken, + applicationTags, this.getApplicationPriority()); report.setLogAggregationStatus(logAggregationStatus); report.setUnmanagedApp(submissionContext.getUnmanagedAM()); report.setAppNodeLabelExpression(getAppNodeLabelExpression()); @@ -831,6 +832,17 @@ public long getStartTime() { } @Override + public long getLaunchTime() { + this.readLock.lock(); + + try { + return this.launchTime; + } finally { + this.readLock.unlock(); + } + } + + @Override public long getSubmitTime() { return this.submitTime; } @@ -1024,6 +1036,7 @@ public RMAppStateUpdateTransition(YarnApplicationState state) { } public void transition(RMAppImpl app, RMAppEvent event) { + app.launchTime = app.systemClock.getTime(); app.rmContext.getSystemMetricsPublisher().appStateUpdated( app, stateToATS, app.systemClock.getTime()); }; diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/FairSchedulerAppsBlock.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/FairSchedulerAppsBlock.java index ac88f861feb..43ad835b47b 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/FairSchedulerAppsBlock.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/FairSchedulerAppsBlock.java @@ -88,6 +88,7 @@ public FairSchedulerAppsBlock(ResourceManager rm, ViewContext ctx, th(".type", "Application Type"). th(".queue", "Queue"). th(".fairshare", "Fair Share"). + th(".submittime", "SubmitTime"). th(".starttime", "StartTime"). th(".finishtime", "FinishTime"). th(".state", "State"). @@ -135,6 +136,7 @@ public FairSchedulerAppsBlock(ResourceManager rm, ViewContext ctx, appInfo.getQueue()))).append("\",\"") .append(fairShare).append("\",\"") .append(appInfo.getStartTime()).append("\",\"") + .append(appInfo.getLaunchTime()).append("\",\"") .append(appInfo.getFinishTime()).append("\",\"") .append(appInfo.getState()).append("\",\"") .append(appInfo.getFinalStatus()).append("\",\"") diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMAppsBlock.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMAppsBlock.java index ede71e34378..ce56a9859e7 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMAppsBlock.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMAppsBlock.java @@ -59,6 +59,7 @@ protected void renderData(Block html) { html.table("#apps").thead().tr().th(".id", "ID").th(".user", "User") .th(".name", "Name").th(".type", "Application Type") .th(".queue", "Queue").th(".priority", "Application Priority") + .th(".submittime", "SubmitTime") .th(".starttime", "StartTime") .th(".finishtime", "FinishTime").th(".state", "State") .th(".finalstatus", "FinalStatus") @@ -132,6 +133,7 @@ protected void renderData(Block html) { .getQueue()))).append("\",\"").append(String .valueOf(app.getPriority())) .append("\",\"").append(app.getStartedTime()) + .append("\",\"").append(app.getLaunchedTime()) .append("\",\"").append(app.getFinishedTime()) .append("\",\"") .append(app.getAppState() == null ? UNAVAILABLE : app.getAppState()) diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/AppInfo.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/AppInfo.java index f11939a2f68..1cdffcbc103 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/AppInfo.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/AppInfo.java @@ -86,7 +86,9 @@ // these are only allowed if acls allow protected long startedTime; + protected long launchedTime; protected long finishedTime; + protected long pendingTime; protected long elapsedTime; protected String amContainerLogs; protected String amHostHttpAddress; @@ -177,8 +179,11 @@ public AppInfo(ResourceManager rm, RMApp app, Boolean hasAccess, this.clusterId = ResourceManager.getClusterTimeStamp(); if (hasAccess) { this.startedTime = app.getStartTime(); + this.launchedTime = app.getLaunchTime(); this.finishedTime = app.getFinishTime(); - this.elapsedTime = Times.elapsed(app.getStartTime(), + this.pendingTime = Times.elapsed(app.getStartTime(), + app.getLaunchTime()); + this.elapsedTime = Times.elapsed(app.getLaunchTime(), app.getFinishTime()); this.logAggregationStatus = app.getLogAggregationStatusForAppReport(); RMAppAttempt attempt = app.getCurrentAppAttempt(); @@ -345,6 +350,10 @@ public String getTrackingUrlPretty() { return this.trackingUrlPretty; } + public long getLaunchTime() { + return this.launchedTime; + } + public long getStartTime() { return this.startedTime; } @@ -353,6 +362,10 @@ public long getFinishTime() { return this.finishedTime; } + public long getPendingTime() { + return this.pendingTime; + } + public long getElapsedTime() { return this.elapsedTime; } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/applicationsmanager/MockAsm.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/applicationsmanager/MockAsm.java index 5246eb79a15..c39e62e48dd 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/applicationsmanager/MockAsm.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/applicationsmanager/MockAsm.java @@ -87,7 +87,12 @@ public long getStartTime() { public long getSubmitTime() { throw new UnsupportedOperationException("Not supported yet."); } - + + @Override + public long getLaunchTime() { + throw new UnsupportedOperationException("Not supported yet."); + } + @Override public long getFinishTime() { throw new UnsupportedOperationException("Not supported yet."); @@ -259,6 +264,7 @@ public static RMApp newApplication(int i) { final String name = newAppName(); final String queue = newQueue(); final long start = 123456 + i * 1000; + final long launch = start + i * 100; final long finish = 234567 + i * 1000; final String type = YarnConfiguration.DEFAULT_APPLICATION_TYPE; YarnApplicationState[] allStates = YarnApplicationState.values(); @@ -295,6 +301,11 @@ public long getStartTime() { } @Override + public long getLaunchTime() { + return launch; + } + + @Override public long getFinishTime() { return finish; } @@ -343,7 +354,7 @@ public ApplicationReport createAndGetApplicationReport( ApplicationReport report = ApplicationReport.newInstance( getApplicationId(), appAttemptId, getUser(), getQueue(), getName(), null, 0, null, null, getDiagnostics().toString(), - getTrackingUrl(), getStartTime(), getFinishTime(), + getTrackingUrl(), getStartTime(), getLaunchTime(), getFinishTime(), getFinalApplicationStatus(), usageReport , null, getProgress(), type, null); return report; diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/MockRMApp.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/MockRMApp.java index 9290ff8faa0..7846ea510c9 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/MockRMApp.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/MockRMApp.java @@ -53,6 +53,7 @@ String name = MockApps.newAppName(); String queue = MockApps.newQueue(); long start = System.currentTimeMillis() - (int) (Math.random() * DT); + long launch = start; long submit = start - (int) (Math.random() * DT); long finish = 0; RMAppState state = RMAppState.NEW; @@ -183,6 +184,11 @@ public long getSubmitTime() { return submit; } + @Override + public long getLaunchTime() { + return launch; + } + public void setStartTime(long time) { this.start = time; } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebApp.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebApp.java index 8a6504002a1..e8b5668a2bc 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebApp.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebApp.java @@ -260,7 +260,7 @@ public static ClientRMService mockClientRMService(RMContext rmContext) { app.getName(), (String) null, 0, (Token) null, app.createApplicationState(), app.getDiagnostics().toString(), (String) null, - app.getStartTime(), app.getFinishTime(), + app.getStartTime(), app.getLaunchTime(), app.getFinishTime(), app.getFinalApplicationStatus(), (ApplicationResourceUsageReport) null, app.getTrackingUrl(), app.getProgress(), app.getApplicationType(), (Token) null); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesApps.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesApps.java index 1cbdec3a225..7fd33916095 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesApps.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesApps.java @@ -1478,7 +1478,9 @@ public void verifyAppsXML(NodeList nodes, RMApp app, boolean hasResourceReq) WebServicesTestUtils.getXmlString(element, "diagnostics"), WebServicesTestUtils.getXmlLong(element, "clusterId"), WebServicesTestUtils.getXmlLong(element, "startedTime"), + WebServicesTestUtils.getXmlLong(element, "launchedTime"), WebServicesTestUtils.getXmlLong(element, "finishedTime"), + WebServicesTestUtils.getXmlLong(element, "pendingTime"), WebServicesTestUtils.getXmlLong(element, "elapsedTime"), WebServicesTestUtils.getXmlString(element, "amHostHttpAddress"), WebServicesTestUtils.getXmlString(element, "amContainerLogs"), @@ -1528,7 +1530,7 @@ public void verifyAppsXML(NodeList nodes, RMApp app, boolean hasResourceReq) public void verifyAppInfo(JSONObject info, RMApp app, boolean hasResourceReqs) throws JSONException, Exception { - int expectedNumberOfElements = 36 + (hasResourceReqs ? 2 : 0); + int expectedNumberOfElements = 38 + (hasResourceReqs ? 2 : 0); String appNodeLabelExpression = null; String amNodeLabelExpression = null; if (app.getApplicationSubmissionContext() @@ -1554,7 +1556,8 @@ public void verifyAppInfo(JSONObject info, RMApp app, boolean hasResourceReqs) info.getString("state"), info.getString("finalStatus"), (float) info.getDouble("progress"), info.getString("trackingUI"), info.getString("diagnostics"), info.getLong("clusterId"), - info.getLong("startedTime"), info.getLong("finishedTime"), + info.getLong("startedTime"), info.getLong("launchedTime"), + info.getLong("finishedTime"), info.getLong("pendingTime"), info.getLong("elapsedTime"), info.getString("amHostHttpAddress"), info.getString("amContainerLogs"), info.getInt("allocatedMB"), info.getInt("allocatedVCores"), info.getInt("runningContainers"), @@ -1578,8 +1581,9 @@ public void verifyAppInfo(JSONObject info, RMApp app, boolean hasResourceReqs) public void verifyAppInfoGeneric(RMApp app, String id, String user, String name, String applicationType, String queue, int prioirty, String state, String finalStatus, float progress, String trackingUI, - String diagnostics, long clusterId, long startedTime, long finishedTime, - long elapsedTime, String amHostHttpAddress, String amContainerLogs, + String diagnostics, long clusterId, long startedTime, long launchedTime, + long finishedTime, long pendingTime, long elapsedTime, + String amHostHttpAddress, String amContainerLogs, int allocatedMB, int allocatedVCores, int numContainers, float queueUsagePerc, float clusterUsagePerc, int preemptedResourceMB, int preemptedResourceVCores, @@ -1610,9 +1614,11 @@ public void verifyAppInfoGeneric(RMApp app, String id, String user, assertEquals("clusterId doesn't match", ResourceManager.getClusterTimeStamp(), clusterId); assertEquals("startedTime doesn't match", app.getStartTime(), startedTime); + assertEquals("launchedTime doesn't match", app.getLaunchTime(), launchedTime); assertEquals("finishedTime doesn't match", app.getFinishTime(), finishedTime); - assertTrue("elapsed time not greater than 0", elapsedTime > 0); + assertTrue("pending time not greater than 0", pendingTime > 0); + assertTrue("elapsed time not valid", elapsedTime >= 0); WebServicesTestUtils.checkStringMatch("amHostHttpAddress", app .getCurrentAppAttempt().getMasterContainer().getNodeHttpAddress(), amHostHttpAddress); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/ResourceManagerRest.md b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/ResourceManagerRest.md index 7c1f4188651..9b041d8c372 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/ResourceManagerRest.md +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/ResourceManagerRest.md @@ -1394,7 +1394,9 @@ Response Body: "applicationTags": "", "priority": -1, "startedTime": 1476913457320, + "launchedTime": 1476913472749, "finishedTime": 1476913761898, + "pendingTime": 15429, "elapsedTime": 304578, "amContainerLogs": "http://host.domain.com:8042/node/containerlogs/container_1476912658570_0002_02_000001/user2", "amHostHttpAddress": "host.domain.com:8042", @@ -1470,7 +1472,9 @@ Response Body: "applicationTags": "", "priority": -1, "startedTime": 1476913464750, + "launchedTime": 1476913494913, "finishedTime": 1476913863175, + "pendingTime": 30163, "elapsedTime": 398425, "amContainerLogs": "http://host.domain.com:8042/node/containerlogs/container_1476912658570_0001_02_000001/user1", "amHostHttpAddress": "host.domain.com:8042", @@ -1570,7 +1574,9 @@ Response Body: -1 1476913457320 + 1476913472749 1476913761898 + 15429 304578 http://host.domain.com:8042/node/containerlogs/container_1476912658570_0002_02_000001/user2 host.domain.com:8042 @@ -1645,7 +1651,9 @@ Response Body: -1 1476913464750 + 1476913494913 1476913863175 + 30163 398425 http://host.domain.com:8042/node/containerlogs/container_1476912658570_0001_02_000001/user1 host.domain.com:8042 @@ -1855,8 +1863,10 @@ Note that depending on security settings a user might not be able to see all the | applicationTags | string | Comma separated tags of an application | | priority | string | Priority of the submitted application | | startedTime | long | The time in which application started (in ms since epoch) | +| launchedTime | long | The time at which the application began running (in ms since epoch) | | finishedTime | long | The time in which the application finished (in ms since epoch) | -| elapsedTime | long | The elapsed time since the application started (in ms) | +| pendingTime | long | The elapsed time between the application starting and running (in ms) | +| elapsedTime | long | The elapsed time since the application began running (in ms) | | amContainerLogs | string | The URL of the application master container logs | | amHostHttpAddress | string | The nodes http address of the application master | | amRPCAddress | string | The RPC address of the application master | diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/controllers/app-table-columns.js b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/controllers/app-table-columns.js index 8a34f1a2715..d6745008bfd 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/controllers/app-table-columns.js +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/controllers/app-table-columns.js @@ -73,9 +73,21 @@ export default Ember.Controller.extend({ valueMax: 100 } }, { + id: 'submitTime', + headerTitle: 'Submit Time', + contentPath: 'startTime', + }, { + id: 'pendingTime', + headerTitle: 'Pending Time', + contentPath: 'pendingTime', + cellDefinition: { + type: "duration" + } + }, + { id: 'stTime', headerTitle: 'Start Time', - contentPath: 'startTime', + contentPath: 'validatedLaunchedTs', }, { id: 'elTime', headerTitle: 'Elapsed Time', diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/models/yarn-app.js b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/models/yarn-app.js index 47814e48c2b..fd297a628b6 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/models/yarn-app.js +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/models/yarn-app.js @@ -25,6 +25,8 @@ export default DS.Model.extend({ queue: DS.attr('string'), state: DS.attr('string'), startTime: DS.attr('string'), + pendingTime: DS.attr('string'), + launchedTime: DS.attr('string'), elapsedTime: DS.attr('string'), finalStatus: DS.attr('string'), finishedTime: DS.attr('finishedTime'), @@ -57,17 +59,28 @@ export default DS.Model.extend({ return this.get('finalStatus') === "FAILED"; }.property("finalStatus"), - validatedFinishedTs: function() { - if (this.get("finishedTime") < this.get("startTime")) { + formattedPendingTime: function() { + return Converter.msToElapsedTimeUnit(this.get('pendingTime')); + }.property('pendingTime'), + + validatedLaunchedTs: function() { + if (this.get("launchedTime") < this.get("startTime")) { return "N/A"; } - return this.get("finishedTime"); - }.property("finishedTime"), + return this.get("launchedTime"); + }.property("launchedTime"), formattedElapsedTime: function() { return Converter.msToElapsedTimeUnit(this.get('elapsedTime')); }.property('elapsedTime'), + validatedFinishedTs: function() { + if (this.get("finishedTime") < this.get("startTime")) { + return "N/A"; + } + return this.get("finishedTime"); + }.property("finishedTime"), + allocatedResource: function() { return Converter.resourceToString(this.get("allocatedMB"), this.get("allocatedVCores")); }.property("allocatedMB", "allocatedVCores"), diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/serializers/yarn-app.js b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/serializers/yarn-app.js index 1462f5aa085..3a862e10a2d 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/serializers/yarn-app.js +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/serializers/yarn-app.js @@ -43,6 +43,8 @@ export default DS.JSONAPISerializer.extend({ queue: payload.queue, state: payload.state, startTime: Converter.timeStampToDate(payload.startedTime), + pendingTime: payload.pendingTime, + launchedTime: Converter.timeStampToDate(payload.launchedTime), elapsedTime: payload.elapsedTime, finishedTime: Converter.timeStampToDate(payload.finishedTime), finalStatus: payload.finalStatus, diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/templates/components/app-table.hbs b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/templates/components/app-table.hbs index 3bad063b422..73fc6eb7bb8 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/templates/components/app-table.hbs +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/templates/components/app-table.hbs @@ -26,6 +26,8 @@ Queue State Final Status + Submit Time + Pending Time Start Time Elapsed Time Finished Time @@ -46,6 +48,8 @@ {{app.state}} {{app.finalStatus}} {{app.startTime}} + {{app.pendingTime}} + {{app.validatedLaunchedTs}} {{app.elapsedTime}} {{app.validatedFinishedTs}} {{app.priority}} @@ -69,6 +73,8 @@ {{app.state}} {{app.finalStatus}} {{app.startTime}} + {{app.pendingTime}} + {{app.validatedLaunchedTs}} {{app.elapsedTime}} {{app.validatedFinishedTs}} {{app.priority}} diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/templates/yarn-app/info.hbs b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/templates/yarn-app/info.hbs index 3cfec338699..a76d677337a 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/templates/yarn-app/info.hbs +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/templates/yarn-app/info.hbs @@ -39,6 +39,8 @@ Queue State Final Status + Submit Time + Pending Time Start Time Elapsed Time Finished Time @@ -63,6 +65,8 @@ {{model.app.startTime}} + {{model.app.formattedPendingTime}} + {{model.app.validatedLaunchedTs}} {{model.app.formattedElapsedTime}} {{model.app.validatedFinishedTs}} {{model.app.priority}} -- 2.11.0 (Apple Git-81)