From b334d1046740faea941494cfd5f16baa6c9c5bbd Mon Sep 17 00:00:00 2001 From: Prabhu Joseph Date: Sat, 26 Jan 2019 08:29:38 +0530 Subject: [PATCH] YARN-6616-branch-2.7.0001.patch --- .../hadoop/yarn/api/records/ApplicationReport.java | 31 ++++++++++++++++++++++ .../src/main/proto/yarn_protos.proto | 1 + .../records/impl/pb/ApplicationReportPBImpl.java | 12 +++++++++ .../ApplicationHistoryManagerImpl.java | 6 ++--- .../ApplicationHistoryManagerOnTimelineStore.java | 14 +++++++--- .../hadoop/yarn/server/webapp/dao/AppInfo.java | 2 +- .../applicationsmanager/MockAsm.java | 8 +++++- 7 files changed, 65 insertions(+), 9 deletions(-) diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ApplicationReport.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ApplicationReport.java index ff4fb52..9c420ca 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ApplicationReport.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ApplicationReport.java @@ -83,6 +83,25 @@ public static ApplicationReport newInstance(ApplicationId applicationId, return report; } + @Private + @Unstable + public static ApplicationReport newInstance(ApplicationId applicationId, + ApplicationAttemptId applicationAttemptId, String user, String queue, + String name, String host, int rpcPort, Token clientToAMToken, + YarnApplicationState state, String diagnostics, String url, + long startTime, long submitTime, long finishTime, + FinalApplicationStatus finalStatus, + ApplicationResourceUsageReport appResources, String origTrackingUrl, + float progress, String applicationType, Token amRmToken) { + ApplicationReport report = + newInstance(applicationId, applicationAttemptId, user, queue, name, + host, rpcPort, clientToAMToken, state, diagnostics, url, startTime, + finishTime, finalStatus, appResources, origTrackingUrl, progress, + applicationType, amRmToken); + report.setSubmitTime(submitTime); + return report; + } + /** * Get the ApplicationId of the application. * @return ApplicationId of the application @@ -258,6 +277,18 @@ public static ApplicationReport newInstance(ApplicationId applicationId, public abstract void setStartTime(long startTime); /** + * Get the submit time of the application. + * @return submit time of the application + */ + @Public + @Stable + public abstract long getSubmitTime(); + + @Private + @Unstable + public abstract void setSubmitTime(long submitTime); + + /** * Get the finish time of the application. * @return finish time of the application */ diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/yarn_protos.proto b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/yarn_protos.proto index 2edff99..5d075c6 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/yarn_protos.proto +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/yarn_protos.proto @@ -194,6 +194,7 @@ message ApplicationReportProto { optional string applicationType = 18; optional hadoop.common.TokenProto am_rm_token = 19; repeated string applicationTags = 20; + optional int64 submitTime = 21; } message ApplicationAttemptReportProto { diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ApplicationReportPBImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ApplicationReportPBImpl.java index dd3e2bc..24b1551 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ApplicationReportPBImpl.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ApplicationReportPBImpl.java @@ -207,6 +207,12 @@ public long getStartTime() { } @Override + public long getSubmitTime() { + ApplicationReportProtoOrBuilder p = viaProto ? proto : builder; + return p.getSubmitTime(); + } + + @Override public long getFinishTime() { ApplicationReportProtoOrBuilder p = viaProto ? proto : builder; return p.getFinishTime(); @@ -400,6 +406,12 @@ public void setStartTime(long startTime) { } @Override + public void setSubmitTime(long submitTime) { + maybeInitBuilder(); + builder.setSubmitTime(submitTime); + } + + @Override public void setFinishTime(long finishTime) { maybeInitBuilder(); builder.setFinishTime(finishTime); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/ApplicationHistoryManagerImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/ApplicationHistoryManagerImpl.java index 479858f..bb264d6 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/ApplicationHistoryManagerImpl.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/ApplicationHistoryManagerImpl.java @@ -137,9 +137,9 @@ private ApplicationReport convertToApplicationReport( currentApplicationAttemptId, appHistory.getUser(), appHistory.getQueue(), appHistory.getApplicationName(), host, rpcPort, null, appHistory.getYarnApplicationState(), appHistory.getDiagnosticsInfo(), - trackingUrl, appHistory.getStartTime(), appHistory.getFinishTime(), - appHistory.getFinalApplicationStatus(), null, "", 100, - appHistory.getApplicationType(), null); + trackingUrl, appHistory.getStartTime(), appHistory.getSubmitTime(), + appHistory.getFinishTime(), appHistory.getFinalApplicationStatus(), null, + "", 100, appHistory.getApplicationType(), null); } private ApplicationAttemptHistoryData getLastAttempt(ApplicationId appId) diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/ApplicationHistoryManagerOnTimelineStore.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/ApplicationHistoryManagerOnTimelineStore.java index c177112..473914c 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/ApplicationHistoryManagerOnTimelineStore.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/ApplicationHistoryManagerOnTimelineStore.java @@ -242,6 +242,7 @@ private static ApplicationReportExt convertToApplicationReport( String name = null; String type = null; long createdTime = 0; + long submittedTime = 0; long finishedTime = 0; ApplicationAttemptId latestApplicationAttemptId = null; String diagnosticsInfo = null; @@ -268,8 +269,8 @@ private static ApplicationReportExt convertToApplicationReport( return new ApplicationReportExt(ApplicationReport.newInstance( ConverterUtils.toApplicationId(entity.getEntityId()), latestApplicationAttemptId, user, queue, name, null, -1, null, state, - diagnosticsInfo, null, createdTime, finishedTime, finalStatus, null, - null, 1.0F, type, null), appViewACLs); + diagnosticsInfo, null, createdTime, submittedTime, finishedTime, + finalStatus, null, null, 1.0F, type, null), appViewACLs); } if (entityInfo.containsKey(ApplicationMetricsConstants.QUEUE_ENTITY_INFO)) { queue = @@ -286,6 +287,11 @@ private static ApplicationReportExt convertToApplicationReport( entityInfo.get(ApplicationMetricsConstants.TYPE_ENTITY_INFO) .toString(); } + if (entityInfo.containsKey(ApplicationMetricsConstants. + SUBMITTED_TIME_ENTITY_INFO)) { + submittedTime = Long.parseLong(entityInfo.get( + ApplicationMetricsConstants.SUBMITTED_TIME_ENTITY_INFO).toString()); + } if (entityInfo.containsKey(ApplicationMetricsConstants.APP_CPU_METRICS)) { long vcoreSeconds=Long.parseLong(entityInfo.get( ApplicationMetricsConstants.APP_CPU_METRICS).toString()); @@ -344,8 +350,8 @@ private static ApplicationReportExt convertToApplicationReport( return new ApplicationReportExt(ApplicationReport.newInstance( ConverterUtils.toApplicationId(entity.getEntityId()), latestApplicationAttemptId, user, queue, name, null, -1, null, state, - diagnosticsInfo, null, createdTime, finishedTime, finalStatus, appResources, - null, 1.0F, type, null), appViewACLs); + diagnosticsInfo, null, createdTime, submittedTime, finishedTime, + finalStatus, appResources, null, 1.0F, type, null), appViewACLs); } private static ApplicationAttemptReport convertToApplicationAttemptReport( diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/webapp/dao/AppInfo.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/webapp/dao/AppInfo.java index 0e17e3d..de14e03 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/webapp/dao/AppInfo.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/webapp/dao/AppInfo.java @@ -77,7 +77,7 @@ public AppInfo(ApplicationReport app) { diagnosticsInfo = app.getDiagnostics(); trackingUrl = app.getTrackingUrl(); originalTrackingUrl = app.getOriginalTrackingUrl(); - submittedTime = app.getStartTime(); + submittedTime = app.getSubmitTime(); startedTime = app.getStartTime(); finishedTime = app.getFinishTime(); elapsedTime = Times.elapsed(startedTime, finishedTime); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/applicationsmanager/MockAsm.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/applicationsmanager/MockAsm.java index f8d92aa..fde3972 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/applicationsmanager/MockAsm.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/applicationsmanager/MockAsm.java @@ -203,6 +203,7 @@ public static RMApp newApplication(int i) { final String name = newAppName(); final String queue = newQueue(); final long start = 123456 + i * 1000; + final long submit = start + i * 50; final long finish = 234567 + i * 1000; final String type = YarnConfiguration.DEFAULT_APPLICATION_TYPE; YarnApplicationState[] allStates = YarnApplicationState.values(); @@ -239,6 +240,11 @@ public long getStartTime() { } @Override + public long getSubmitTime() { + return submit; + } + + @Override public long getFinishTime() { return finish; } @@ -287,7 +293,7 @@ public ApplicationReport createAndGetApplicationReport( ApplicationReport report = ApplicationReport.newInstance( getApplicationId(), appAttemptId, getUser(), getQueue(), getName(), null, 0, null, null, getDiagnostics().toString(), - getTrackingUrl(), getStartTime(), getFinishTime(), + getTrackingUrl(), getStartTime(), getSubmitTime(), getFinishTime(), getFinalApplicationStatus(), usageReport, null, getProgress(), type, null); return report; -- 2.7.4 (Apple Git-66)