From 640d9bf171cddebe6c6d6525e72dadb492ad1ac3 Mon Sep 17 00:00:00 2001 From: Prabhu Joseph Date: Sat, 26 Jan 2019 17:58:00 +0530 Subject: [PATCH] YARN-6616-branch-2.7.003.patch --- .../hadoop/yarn/api/records/ApplicationReport.java | 31 +++++++++++++++++++++- .../src/main/proto/yarn_protos.proto | 1 + .../records/impl/pb/ApplicationReportPBImpl.java | 12 +++++++++ .../ApplicationHistoryManagerImpl.java | 6 ++--- .../ApplicationHistoryManagerOnTimelineStore.java | 15 ++++++++--- .../hadoop/yarn/server/webapp/dao/AppInfo.java | 2 +- .../applicationsmanager/MockAsm.java | 16 +++++++---- 7 files changed, 69 insertions(+), 14 deletions(-) diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ApplicationReport.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ApplicationReport.java index ff4fb52..d79d54d 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ApplicationReport.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ApplicationReport.java @@ -57,7 +57,8 @@ public static ApplicationReport newInstance(ApplicationId applicationId, ApplicationAttemptId applicationAttemptId, String user, String queue, String name, String host, int rpcPort, Token clientToAMToken, YarnApplicationState state, String diagnostics, String url, - long startTime, long finishTime, FinalApplicationStatus finalStatus, + long startTime, long submitTime, long finishTime, + FinalApplicationStatus finalStatus, ApplicationResourceUsageReport appResources, String origTrackingUrl, float progress, String applicationType, Token amRmToken) { ApplicationReport report = Records.newRecord(ApplicationReport.class); @@ -73,6 +74,7 @@ public static ApplicationReport newInstance(ApplicationId applicationId, report.setDiagnostics(diagnostics); report.setTrackingUrl(url); report.setStartTime(startTime); + report.setSubmitTime(submitTime); report.setFinishTime(finishTime); report.setFinalApplicationStatus(finalStatus); report.setApplicationResourceUsageReport(appResources); @@ -83,6 +85,21 @@ public static ApplicationReport newInstance(ApplicationId applicationId, return report; } + @Private + @Unstable + public static ApplicationReport newInstance(ApplicationId applicationId, + ApplicationAttemptId applicationAttemptId, String user, String queue, + String name, String host, int rpcPort, Token clientToAMToken, + YarnApplicationState state, String diagnostics, String url, + long startTime, long finishTime, FinalApplicationStatus finalStatus, + ApplicationResourceUsageReport appResources, String origTrackingUrl, + float progress, String applicationType, Token amRmToken) { + return newInstance(applicationId, applicationAttemptId, user, queue, name, + host, rpcPort, clientToAMToken, state, diagnostics, url, startTime, + startTime, finishTime, finalStatus, appResources, origTrackingUrl, + progress, applicationType, amRmToken); + } + /** * Get the ApplicationId of the application. * @return ApplicationId of the application @@ -258,6 +275,18 @@ public static ApplicationReport newInstance(ApplicationId applicationId, public abstract void setStartTime(long startTime); /** + * Get the submit time of the application. + * @return submit time of the application + */ + @Public + @Stable + public abstract long getSubmitTime(); + + @Private + @Unstable + public abstract void setSubmitTime(long submitTime); + + /** * Get the finish time of the application. * @return finish time of the application */ diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/yarn_protos.proto b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/yarn_protos.proto index 2edff99..5d075c6 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/yarn_protos.proto +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/yarn_protos.proto @@ -194,6 +194,7 @@ message ApplicationReportProto { optional string applicationType = 18; optional hadoop.common.TokenProto am_rm_token = 19; repeated string applicationTags = 20; + optional int64 submitTime = 21; } message ApplicationAttemptReportProto { diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ApplicationReportPBImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ApplicationReportPBImpl.java index dd3e2bc..24b1551 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ApplicationReportPBImpl.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ApplicationReportPBImpl.java @@ -207,6 +207,12 @@ public long getStartTime() { } @Override + public long getSubmitTime() { + ApplicationReportProtoOrBuilder p = viaProto ? proto : builder; + return p.getSubmitTime(); + } + + @Override public long getFinishTime() { ApplicationReportProtoOrBuilder p = viaProto ? proto : builder; return p.getFinishTime(); @@ -400,6 +406,12 @@ public void setStartTime(long startTime) { } @Override + public void setSubmitTime(long submitTime) { + maybeInitBuilder(); + builder.setSubmitTime(submitTime); + } + + @Override public void setFinishTime(long finishTime) { maybeInitBuilder(); builder.setFinishTime(finishTime); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/ApplicationHistoryManagerImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/ApplicationHistoryManagerImpl.java index 479858f..aadb0fd 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/ApplicationHistoryManagerImpl.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/ApplicationHistoryManagerImpl.java @@ -137,9 +137,9 @@ private ApplicationReport convertToApplicationReport( currentApplicationAttemptId, appHistory.getUser(), appHistory.getQueue(), appHistory.getApplicationName(), host, rpcPort, null, appHistory.getYarnApplicationState(), appHistory.getDiagnosticsInfo(), - trackingUrl, appHistory.getStartTime(), appHistory.getFinishTime(), - appHistory.getFinalApplicationStatus(), null, "", 100, - appHistory.getApplicationType(), null); + trackingUrl, appHistory.getStartTime(), appHistory.getSubmitTime(), + appHistory.getFinishTime(), appHistory.getFinalApplicationStatus(), + null, "", 100, appHistory.getApplicationType(), null); } private ApplicationAttemptHistoryData getLastAttempt(ApplicationId appId) diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/ApplicationHistoryManagerOnTimelineStore.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/ApplicationHistoryManagerOnTimelineStore.java index c177112..0cec43a 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/ApplicationHistoryManagerOnTimelineStore.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/ApplicationHistoryManagerOnTimelineStore.java @@ -242,6 +242,7 @@ private static ApplicationReportExt convertToApplicationReport( String name = null; String type = null; long createdTime = 0; + long submittedTime = 0; long finishedTime = 0; ApplicationAttemptId latestApplicationAttemptId = null; String diagnosticsInfo = null; @@ -268,8 +269,8 @@ private static ApplicationReportExt convertToApplicationReport( return new ApplicationReportExt(ApplicationReport.newInstance( ConverterUtils.toApplicationId(entity.getEntityId()), latestApplicationAttemptId, user, queue, name, null, -1, null, state, - diagnosticsInfo, null, createdTime, finishedTime, finalStatus, null, - null, 1.0F, type, null), appViewACLs); + diagnosticsInfo, null, createdTime, submittedTime, finishedTime, + finalStatus, null, null, 1.0F, type, null), appViewACLs); } if (entityInfo.containsKey(ApplicationMetricsConstants.QUEUE_ENTITY_INFO)) { queue = @@ -286,6 +287,12 @@ private static ApplicationReportExt convertToApplicationReport( entityInfo.get(ApplicationMetricsConstants.TYPE_ENTITY_INFO) .toString(); } + if (entityInfo.containsKey(ApplicationMetricsConstants. + SUBMITTED_TIME_ENTITY_INFO)) { + submittedTime = Long.parseLong(entityInfo.get( + ApplicationMetricsConstants.SUBMITTED_TIME_ENTITY_INFO) + .toString()); + } if (entityInfo.containsKey(ApplicationMetricsConstants.APP_CPU_METRICS)) { long vcoreSeconds=Long.parseLong(entityInfo.get( ApplicationMetricsConstants.APP_CPU_METRICS).toString()); @@ -344,8 +351,8 @@ private static ApplicationReportExt convertToApplicationReport( return new ApplicationReportExt(ApplicationReport.newInstance( ConverterUtils.toApplicationId(entity.getEntityId()), latestApplicationAttemptId, user, queue, name, null, -1, null, state, - diagnosticsInfo, null, createdTime, finishedTime, finalStatus, appResources, - null, 1.0F, type, null), appViewACLs); + diagnosticsInfo, null, createdTime, submittedTime, finishedTime, + finalStatus, appResources, null, 1.0F, type, null), appViewACLs); } private static ApplicationAttemptReport convertToApplicationAttemptReport( diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/webapp/dao/AppInfo.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/webapp/dao/AppInfo.java index 0e17e3d..de14e03 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/webapp/dao/AppInfo.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/webapp/dao/AppInfo.java @@ -77,7 +77,7 @@ public AppInfo(ApplicationReport app) { diagnosticsInfo = app.getDiagnostics(); trackingUrl = app.getTrackingUrl(); originalTrackingUrl = app.getOriginalTrackingUrl(); - submittedTime = app.getStartTime(); + submittedTime = app.getSubmitTime(); startedTime = app.getStartTime(); finishedTime = app.getFinishTime(); elapsedTime = Times.elapsed(startedTime, finishedTime); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/applicationsmanager/MockAsm.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/applicationsmanager/MockAsm.java index f8d92aa..6ef8d1e 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/applicationsmanager/MockAsm.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/applicationsmanager/MockAsm.java @@ -203,6 +203,7 @@ public static RMApp newApplication(int i) { final String name = newAppName(); final String queue = newQueue(); final long start = 123456 + i * 1000; + final long submit = start + i * 50; final long finish = 234567 + i * 1000; final String type = YarnConfiguration.DEFAULT_APPLICATION_TYPE; YarnApplicationState[] allStates = YarnApplicationState.values(); @@ -239,6 +240,11 @@ public long getStartTime() { } @Override + public long getSubmitTime() { + return submit; + } + + @Override public long getFinishTime() { return finish; } @@ -282,13 +288,13 @@ public int getMaxAppAttempts() { public ApplicationReport createAndGetApplicationReport( String clientUserName, boolean allowAccess) { ApplicationResourceUsageReport usageReport = - ApplicationResourceUsageReport.newInstance(0, 0, null, null, null, + ApplicationResourceUsageReport.newInstance(0, 0, null, null, null, 0, 0); ApplicationReport report = ApplicationReport.newInstance( - getApplicationId(), appAttemptId, getUser(), getQueue(), - getName(), null, 0, null, null, getDiagnostics().toString(), - getTrackingUrl(), getStartTime(), getFinishTime(), - getFinalApplicationStatus(), usageReport , null, getProgress(), + getApplicationId(), appAttemptId, getUser(), getQueue(), + getName(), null, 0, null, null, getDiagnostics().toString(), + getTrackingUrl(), getStartTime(), getSubmitTime(), getFinishTime(), + getFinalApplicationStatus(), usageReport, null, getProgress(), type, null); return report; } -- 2.7.4 (Apple Git-66)