From efb4330771608c21501fb22e4da38b72ec0b025c Mon Sep 17 00:00:00 2001 From: Prabhu Joseph Date: Sat, 26 Jan 2019 14:37:59 +0530 Subject: [PATCH] YARN-6616-branch-2.8.002.patch --- .../hadoop/yarn/api/records/ApplicationReport.java | 55 ++++++++++++++++++++-- .../src/main/proto/yarn_protos.proto | 1 + .../records/impl/pb/ApplicationReportPBImpl.java | 12 +++++ .../ApplicationHistoryManagerImpl.java | 6 +-- .../ApplicationHistoryManagerOnTimelineStore.java | 20 ++++---- .../hadoop/yarn/server/webapp/dao/AppInfo.java | 2 +- .../applicationsmanager/MockAsm.java | 14 ++++-- 7 files changed, 91 insertions(+), 19 deletions(-) diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ApplicationReport.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ApplicationReport.java index 33116a4..02056ca 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ApplicationReport.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ApplicationReport.java @@ -57,7 +57,8 @@ public static ApplicationReport newInstance(ApplicationId applicationId, ApplicationAttemptId applicationAttemptId, String user, String queue, String name, String host, int rpcPort, Token clientToAMToken, YarnApplicationState state, String diagnostics, String url, - long startTime, long finishTime, FinalApplicationStatus finalStatus, + long startTime, long submitTime, long finishTime, + FinalApplicationStatus finalStatus, ApplicationResourceUsageReport appResources, String origTrackingUrl, float progress, String applicationType, Token amRmToken) { ApplicationReport report = Records.newRecord(ApplicationReport.class); @@ -73,6 +74,7 @@ public static ApplicationReport newInstance(ApplicationId applicationId, report.setDiagnostics(diagnostics); report.setTrackingUrl(url); report.setStartTime(startTime); + report.setSubmitTime(submitTime); report.setFinishTime(finishTime); report.setFinalApplicationStatus(finalStatus); report.setApplicationResourceUsageReport(appResources); @@ -91,14 +93,30 @@ public static ApplicationReport newInstance(ApplicationId applicationId, YarnApplicationState state, String diagnostics, String url, long startTime, long finishTime, FinalApplicationStatus finalStatus, ApplicationResourceUsageReport appResources, String origTrackingUrl, + float progress, String applicationType, Token amRmToken) { + return newInstance(applicationId, applicationAttemptId, user, queue, + name, host, rpcPort, clientToAMToken, state, diagnostics, url, + startTime, startTime, finishTime, finalStatus, appResources, + origTrackingUrl, progress, applicationType, amRmToken); + } + + @Private + @Unstable + public static ApplicationReport newInstance(ApplicationId applicationId, + ApplicationAttemptId applicationAttemptId, String user, String queue, + String name, String host, int rpcPort, Token clientToAMToken, + YarnApplicationState state, String diagnostics, String url, + long startTime, long submitTime, long finishTime, + FinalApplicationStatus finalStatus, + ApplicationResourceUsageReport appResources, String origTrackingUrl, float progress, String applicationType, Token amRmToken, Set tags, boolean unmanagedApplication, Priority priority, String appNodeLabelExpression, String amNodeLabelExpression) { ApplicationReport report = newInstance(applicationId, applicationAttemptId, user, queue, name, host, rpcPort, clientToAMToken, state, diagnostics, url, startTime, - finishTime, finalStatus, appResources, origTrackingUrl, progress, - applicationType, amRmToken); + submitTime, finishTime, finalStatus, appResources, origTrackingUrl, + progress, applicationType, amRmToken); report.setApplicationTags(tags); report.setUnmanagedApp(unmanagedApplication); report.setPriority(priority); @@ -107,6 +125,25 @@ public static ApplicationReport newInstance(ApplicationId applicationId, return report; } + @Private + @Unstable + public static ApplicationReport newInstance(ApplicationId applicationId, + ApplicationAttemptId applicationAttemptId, String user, String queue, + String name, String host, int rpcPort, Token clientToAMToken, + YarnApplicationState state, String diagnostics, String url, + long startTime, long finishTime, FinalApplicationStatus finalStatus, + ApplicationResourceUsageReport appResources, String origTrackingUrl, + float progress, String applicationType, Token amRmToken, Set tags, + boolean unmanagedApplication, Priority priority, + String appNodeLabelExpression, String amNodeLabelExpression) { + return newInstance(applicationId, applicationAttemptId, user, queue, + name, host, rpcPort, clientToAMToken, state, diagnostics, url, + startTime, startTime, finishTime, finalStatus, appResources, + origTrackingUrl, progress, applicationType, amRmToken, tags, + unmanagedApplication, priority, appNodeLabelExpression, + amNodeLabelExpression); + } + /** * Get the ApplicationId of the application. * @return ApplicationId of the application @@ -282,6 +319,18 @@ public static ApplicationReport newInstance(ApplicationId applicationId, public abstract void setStartTime(long startTime); /** + * Get the submit time of the application. + * @return submit time of the application + */ + @Public + @Stable + public abstract long getSubmitTime(); + + @Private + @Unstable + public abstract void setSubmitTime(long submitTime); + + /** * Get the finish time of the application. * @return finish time of the application */ diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/yarn_protos.proto b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/yarn_protos.proto index 16e7c5b..448d289 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/yarn_protos.proto +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/yarn_protos.proto @@ -211,6 +211,7 @@ message ApplicationReportProto { optional PriorityProto priority = 23; optional string appNodeLabelExpression = 24; optional string amNodeLabelExpression = 25; + optional int64 submitTime = 26; } enum LogAggregationStatusProto { diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ApplicationReportPBImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ApplicationReportPBImpl.java index 1072815..48553f4 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ApplicationReportPBImpl.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ApplicationReportPBImpl.java @@ -212,6 +212,12 @@ public long getStartTime() { } @Override + public long getSubmitTime() { + ApplicationReportProtoOrBuilder p = viaProto ? proto : builder; + return p.getSubmitTime(); + } + + @Override public long getFinishTime() { ApplicationReportProtoOrBuilder p = viaProto ? proto : builder; return p.getFinishTime(); @@ -405,6 +411,12 @@ public void setStartTime(long startTime) { } @Override + public void setSubmitTime(long submitTime) { + maybeInitBuilder(); + builder.setSubmitTime(submitTime); + } + + @Override public void setFinishTime(long finishTime) { maybeInitBuilder(); builder.setFinishTime(finishTime); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/ApplicationHistoryManagerImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/ApplicationHistoryManagerImpl.java index 130bb32..c8ffb75 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/ApplicationHistoryManagerImpl.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/ApplicationHistoryManagerImpl.java @@ -137,9 +137,9 @@ private ApplicationReport convertToApplicationReport( currentApplicationAttemptId, appHistory.getUser(), appHistory.getQueue(), appHistory.getApplicationName(), host, rpcPort, null, appHistory.getYarnApplicationState(), appHistory.getDiagnosticsInfo(), - trackingUrl, appHistory.getStartTime(), appHistory.getFinishTime(), - appHistory.getFinalApplicationStatus(), null, "", 100, - appHistory.getApplicationType(), null); + trackingUrl, appHistory.getStartTime(), appHistory.getSubmitTime(), + appHistory.getFinishTime(), appHistory.getFinalApplicationStatus(), + null, "", 100, appHistory.getApplicationType(), null); } private ApplicationAttemptHistoryData getLastAttempt(ApplicationId appId) diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/ApplicationHistoryManagerOnTimelineStore.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/ApplicationHistoryManagerOnTimelineStore.java index 7434178..5093828 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/ApplicationHistoryManagerOnTimelineStore.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/ApplicationHistoryManagerOnTimelineStore.java @@ -249,6 +249,7 @@ private static ApplicationReportExt convertToApplicationReport( String type = null; boolean unmanagedApplication = false; long createdTime = 0; + long submittedTime = 0; long finishedTime = 0; float progress = 0.0f; int applicationPriority = 0; @@ -280,10 +281,11 @@ private static ApplicationReportExt convertToApplicationReport( return new ApplicationReportExt(ApplicationReport.newInstance( ApplicationId.fromString(entity.getEntityId()), latestApplicationAttemptId, user, queue, name, null, -1, null, - state, diagnosticsInfo, null, createdTime, finishedTime, - finalStatus, null, null, progress, type, null, appTags, - unmanagedApplication, Priority.newInstance(applicationPriority), - appNodeLabelExpression, amNodeLabelExpression), appViewACLs); + state, diagnosticsInfo, null, createdTime, submittedTime, + finishedTime, finalStatus, null, null, progress, type, null, + appTags, unmanagedApplication, Priority.newInstance( + applicationPriority), appNodeLabelExpression, + amNodeLabelExpression), appViewACLs); } if (entityInfo.containsKey(ApplicationMetricsConstants.QUEUE_ENTITY_INFO)) { queue = @@ -328,6 +330,8 @@ private static ApplicationReportExt convertToApplicationReport( entityInfo.get(ApplicationMetricsConstants.AM_NODE_LABEL_EXPRESSION) .toString(); } + submittedTime = parseLong(entityInfo, + ApplicationMetricsConstants.SUBMITTED_TIME_ENTITY_INFO); if (entityInfo.containsKey(ApplicationMetricsConstants.APP_CPU_METRICS)) { long vcoreSeconds = parseLong(entityInfo, @@ -438,10 +442,10 @@ private static ApplicationReportExt convertToApplicationReport( return new ApplicationReportExt(ApplicationReport.newInstance( ApplicationId.fromString(entity.getEntityId()), latestApplicationAttemptId, user, queue, name, null, -1, null, state, - diagnosticsInfo, null, createdTime, finishedTime, finalStatus, - appResources, null, progress, type, null, appTags, unmanagedApplication, - Priority.newInstance(applicationPriority), appNodeLabelExpression, - amNodeLabelExpression), appViewACLs); + diagnosticsInfo, null, createdTime, submittedTime, finishedTime, + finalStatus, appResources, null, progress, type, null, appTags, + unmanagedApplication, Priority.newInstance(applicationPriority), + appNodeLabelExpression, amNodeLabelExpression), appViewACLs); } private static long parseLong(Map entityInfo, diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/webapp/dao/AppInfo.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/webapp/dao/AppInfo.java index ff4519d..5eb5bcd 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/webapp/dao/AppInfo.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/webapp/dao/AppInfo.java @@ -84,7 +84,7 @@ public AppInfo(ApplicationReport app) { diagnosticsInfo = app.getDiagnostics(); trackingUrl = app.getTrackingUrl(); originalTrackingUrl = app.getOriginalTrackingUrl(); - submittedTime = app.getStartTime(); + submittedTime = app.getSubmitTime(); startedTime = app.getStartTime(); finishedTime = app.getFinishTime(); elapsedTime = Times.elapsed(startedTime, finishedTime); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/applicationsmanager/MockAsm.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/applicationsmanager/MockAsm.java index 4df2c42..2e4b012 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/applicationsmanager/MockAsm.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/applicationsmanager/MockAsm.java @@ -230,6 +230,7 @@ public static RMApp newApplication(int i) { final String name = newAppName(); final String queue = newQueue(); final long start = 123456 + i * 1000; + final long submit = start + i * 50; final long finish = 234567 + i * 1000; final String type = YarnConfiguration.DEFAULT_APPLICATION_TYPE; YarnApplicationState[] allStates = YarnApplicationState.values(); @@ -266,6 +267,11 @@ public long getStartTime() { } @Override + public long getSubmitTime() { + return submit; + } + + @Override public long getFinishTime() { return finish; } @@ -312,10 +318,10 @@ public ApplicationReport createAndGetApplicationReport( ApplicationResourceUsageReport.newInstance(0, 0, null, null, null, 0, 0, 0, 0, 0, 0); ApplicationReport report = ApplicationReport.newInstance( - getApplicationId(), appAttemptId, getUser(), getQueue(), - getName(), null, 0, null, null, getDiagnostics().toString(), - getTrackingUrl(), getStartTime(), getFinishTime(), - getFinalApplicationStatus(), usageReport , null, getProgress(), + getApplicationId(), appAttemptId, getUser(), getQueue(), + getName(), null, 0, null, null, getDiagnostics().toString(), + getTrackingUrl(), getStartTime(), getSubmitTime(), getFinishTime(), + getFinalApplicationStatus(), usageReport, null, getProgress(), type, null); return report; } -- 2.7.4 (Apple Git-66)