From 0d3cdad1ed55689e2cd93ca27218e2b31629322a Mon Sep 17 00:00:00 2001 From: Prabhu Joseph Date: Wed, 23 Jan 2019 12:20:06 +0530 Subject: [PATCH] YARN-6616 --- .../hadoop/yarn/api/records/ApplicationReport.java | 49 +++++++++++++++++++++- .../src/main/proto/yarn_protos.proto | 1 + .../records/impl/pb/ApplicationReportPBImpl.java | 12 ++++++ .../hadoop/yarn/api/TestApplicatonReport.java | 2 +- .../ApplicationHistoryManagerImpl.java | 6 +-- .../ApplicationHistoryManagerOnTimelineStore.java | 20 +++++---- .../hadoop/yarn/server/webapp/dao/AppInfo.java | 4 +- .../applicationsmanager/MockAsm.java | 12 ++++-- 8 files changed, 87 insertions(+), 19 deletions(-) diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ApplicationReport.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ApplicationReport.java index 711424b..35f607b 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ApplicationReport.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ApplicationReport.java @@ -62,6 +62,22 @@ public static ApplicationReport newInstance(ApplicationId applicationId, FinalApplicationStatus finalStatus, ApplicationResourceUsageReport appResources, String origTrackingUrl, float progress, String applicationType, Token amRmToken) { + return newInstance(applicationId, applicationAttemptId, user, queue, name, + host, rpcPort, clientToAMToken, state, diagnostics, url, + startTime, startTime, launchTime, finishTime, finalStatus, appResources, + origTrackingUrl, progress, applicationType, amRmToken); + } + + @Private + @Unstable + public static ApplicationReport newInstance(ApplicationId applicationId, + ApplicationAttemptId applicationAttemptId, String user, String queue, + String name, String host, int rpcPort, Token clientToAMToken, + YarnApplicationState state, String diagnostics, String url, + long startTime, long submitTime, long launchTime, long finishTime, + FinalApplicationStatus finalStatus, + ApplicationResourceUsageReport appResources, String origTrackingUrl, + float progress, String applicationType, Token amRmToken) { ApplicationReport report = Records.newRecord(ApplicationReport.class); report.setApplicationId(applicationId); report.setCurrentApplicationAttemptId(applicationAttemptId); @@ -75,6 +91,7 @@ public static ApplicationReport newInstance(ApplicationId applicationId, report.setDiagnostics(diagnostics); report.setTrackingUrl(url); report.setStartTime(startTime); + report.setSubmitTime(submitTime); report.setLaunchTime(launchTime); report.setFinishTime(finishTime); report.setFinalApplicationStatus(finalStatus); @@ -102,7 +119,7 @@ public static ApplicationReport newInstance(ApplicationId applicationId, ApplicationReport report = newInstance(applicationId, applicationAttemptId, user, queue, name, host, rpcPort, clientToAMToken, state, diagnostics, url, - startTime, 0, finishTime, finalStatus, appResources, + startTime, 0, 0, finishTime, finalStatus, appResources, origTrackingUrl, progress, applicationType, amRmToken); report.setApplicationTags(tags); report.setUnmanagedApp(unmanagedApplication); @@ -124,10 +141,30 @@ public static ApplicationReport newInstance(ApplicationId applicationId, float progress, String applicationType, Token amRmToken, Set tags, boolean unmanagedApplication, Priority priority, String appNodeLabelExpression, String amNodeLabelExpression) { + return newInstance(applicationId, applicationAttemptId, user, queue, name, + host, rpcPort, clientToAMToken, state, diagnostics, url, + startTime, startTime, launchTime, finishTime, finalStatus, appResources, + origTrackingUrl, progress, applicationType, amRmToken, tags, + unmanagedApplication, priority, appNodeLabelExpression, + amNodeLabelExpression); + } + + @Private + @Unstable + public static ApplicationReport newInstance(ApplicationId applicationId, + ApplicationAttemptId applicationAttemptId, String user, String queue, + String name, String host, int rpcPort, Token clientToAMToken, + YarnApplicationState state, String diagnostics, String url, + long startTime, long submitTime, long launchTime, long finishTime, + FinalApplicationStatus finalStatus, + ApplicationResourceUsageReport appResources, String origTrackingUrl, + float progress, String applicationType, Token amRmToken, Set tags, + boolean unmanagedApplication, Priority priority, + String appNodeLabelExpression, String amNodeLabelExpression) { ApplicationReport report = newInstance(applicationId, applicationAttemptId, user, queue, name, host, rpcPort, clientToAMToken, state, diagnostics, url, startTime, - launchTime, finishTime, finalStatus, appResources, + submitTime, launchTime, finishTime, finalStatus, appResources, origTrackingUrl, progress, applicationType, amRmToken); report.setApplicationTags(tags); report.setUnmanagedApp(unmanagedApplication); @@ -311,6 +348,14 @@ public static ApplicationReport newInstance(ApplicationId applicationId, @Unstable public abstract void setStartTime(long startTime); + @Public + @Stable + public abstract long getSubmitTime(); + + @Private + @Unstable + public abstract void setSubmitTime(long submitTime); + @Private @Unstable public abstract void setLaunchTime(long setLaunchTime); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/yarn_protos.proto b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/yarn_protos.proto index 8b3071a..4573859 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/yarn_protos.proto +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/yarn_protos.proto @@ -286,6 +286,7 @@ message ApplicationReportProto { optional string amNodeLabelExpression = 25; repeated AppTimeoutsMapProto appTimeouts = 26; optional int64 launchTime = 27; + optional int64 submitTime = 28; } message AppTimeoutsMapProto { diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ApplicationReportPBImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ApplicationReportPBImpl.java index ddd9acc..5209021 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ApplicationReportPBImpl.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ApplicationReportPBImpl.java @@ -221,6 +221,12 @@ public long getStartTime() { } @Override + public long getSubmitTime() { + ApplicationReportProtoOrBuilder p = viaProto ? proto : builder; + return p.getSubmitTime(); + } + + @Override public long getLaunchTime() { ApplicationReportProtoOrBuilder p = viaProto ? proto : builder; return p.getLaunchTime(); @@ -431,6 +437,12 @@ public void setFinishTime(long finishTime) { } @Override + public void setSubmitTime(long submitTime) { + maybeInitBuilder(); + builder.setSubmitTime(submitTime); + } + + @Override public void setFinalApplicationStatus(FinalApplicationStatus finishState) { maybeInitBuilder(); if (finishState == null) { diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/api/TestApplicatonReport.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/api/TestApplicatonReport.java index ea39a4c..8aae1e2 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/api/TestApplicatonReport.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/api/TestApplicatonReport.java @@ -58,7 +58,7 @@ protected static ApplicationReport createApplicationReport( ApplicationReport appReport = ApplicationReport.newInstance(appId, appAttemptId, "user", "queue", "appname", "host", 124, null, YarnApplicationState.FINISHED, - "diagnostics", "url", 0, 0, 0, FinalApplicationStatus.SUCCEEDED, null, + "diagnostics", "url", 0, 0, 0, 0, FinalApplicationStatus.SUCCEEDED, null, "N/A", 0.53789f, YarnConfiguration.DEFAULT_APPLICATION_TYPE, null, null, false, Priority.newInstance(0),"",""); return appReport; diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/ApplicationHistoryManagerImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/ApplicationHistoryManagerImpl.java index 5235164..48a7294 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/ApplicationHistoryManagerImpl.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/ApplicationHistoryManagerImpl.java @@ -137,9 +137,9 @@ private ApplicationReport convertToApplicationReport( currentApplicationAttemptId, appHistory.getUser(), appHistory.getQueue(), appHistory.getApplicationName(), host, rpcPort, null, appHistory.getYarnApplicationState(), appHistory.getDiagnosticsInfo(), - trackingUrl, appHistory.getStartTime(), 0, appHistory.getFinishTime(), - appHistory.getFinalApplicationStatus(), null, "", 100, - appHistory.getApplicationType(), null); + trackingUrl, appHistory.getStartTime(), appHistory.getSubmitTime(), 0, + appHistory.getFinishTime(), appHistory.getFinalApplicationStatus(), + null, "", 100, appHistory.getApplicationType(), null); } private ApplicationAttemptHistoryData getLastAttempt(ApplicationId appId) diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/ApplicationHistoryManagerOnTimelineStore.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/ApplicationHistoryManagerOnTimelineStore.java index 0d31f39..62d8769 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/ApplicationHistoryManagerOnTimelineStore.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/ApplicationHistoryManagerOnTimelineStore.java @@ -250,6 +250,7 @@ private static ApplicationReportExt convertToApplicationReport( String type = null; boolean unmanagedApplication = false; long createdTime = 0; + long submittedTime = 0; long finishedTime = 0; float progress = 0.0f; int applicationPriority = 0; @@ -281,10 +282,11 @@ private static ApplicationReportExt convertToApplicationReport( return new ApplicationReportExt(ApplicationReport.newInstance( ApplicationId.fromString(entity.getEntityId()), latestApplicationAttemptId, user, queue, name, null, -1, null, - state, diagnosticsInfo, null, createdTime, finishedTime, - finalStatus, null, null, progress, type, null, appTags, - unmanagedApplication, Priority.newInstance(applicationPriority), - appNodeLabelExpression, amNodeLabelExpression), appViewACLs); + state, diagnosticsInfo, null, createdTime, submittedTime, 0, + finishedTime, finalStatus, null, null, progress, type, null, + appTags, unmanagedApplication, Priority.newInstance( + applicationPriority), appNodeLabelExpression, + amNodeLabelExpression), appViewACLs); } if (entityInfo.containsKey(ApplicationMetricsConstants.QUEUE_ENTITY_INFO)) { queue = @@ -329,6 +331,8 @@ private static ApplicationReportExt convertToApplicationReport( entityInfo.get(ApplicationMetricsConstants.AM_NODE_LABEL_EXPRESSION) .toString(); } + submittedTime = parseLong(entityInfo, + ApplicationMetricsConstants.SUBMITTED_TIME_ENTITY_INFO); if (entityInfo.containsKey(ApplicationMetricsConstants.APP_CPU_METRICS)) { long vcoreSeconds = parseLong(entityInfo, @@ -450,10 +454,10 @@ private static ApplicationReportExt convertToApplicationReport( return new ApplicationReportExt(ApplicationReport.newInstance( ApplicationId.fromString(entity.getEntityId()), latestApplicationAttemptId, user, queue, name, null, -1, null, state, - diagnosticsInfo, null, createdTime, finishedTime, finalStatus, - appResources, null, progress, type, null, appTags, unmanagedApplication, - Priority.newInstance(applicationPriority), appNodeLabelExpression, - amNodeLabelExpression), appViewACLs); + diagnosticsInfo, null, createdTime, submittedTime, 0, finishedTime, + finalStatus, appResources, null, progress, type, null, appTags, + unmanagedApplication, Priority.newInstance(applicationPriority), + appNodeLabelExpression, amNodeLabelExpression), appViewACLs); } private static long parseLong(Map entityInfo, diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/webapp/dao/AppInfo.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/webapp/dao/AppInfo.java index b4687a4..c4de022 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/webapp/dao/AppInfo.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/webapp/dao/AppInfo.java @@ -53,7 +53,7 @@ protected String originalTrackingUrl; protected String trackingUrl; protected FinalApplicationStatus finalAppStatus; - protected long submittedTime; + private long submittedTime; protected long startedTime; private long launchTime; protected long finishedTime; @@ -87,7 +87,7 @@ public AppInfo(ApplicationReport app) { diagnosticsInfo = app.getDiagnostics(); trackingUrl = app.getTrackingUrl(); originalTrackingUrl = app.getOriginalTrackingUrl(); - submittedTime = app.getStartTime(); + submittedTime = app.getSubmitTime(); startedTime = app.getStartTime(); launchTime = app.getLaunchTime(); finishedTime = app.getFinishTime(); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/applicationsmanager/MockAsm.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/applicationsmanager/MockAsm.java index 342dab8..c679b88 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/applicationsmanager/MockAsm.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/applicationsmanager/MockAsm.java @@ -288,6 +288,7 @@ public static RMApp newApplication(int i) { final String name = newAppName(); final String queue = newQueue(); final long start = 123456 + i * 1000; + final long submit = start + i * 50; final long launch = start + i * 100; final long finish = 234567 + i * 1000; final String type = YarnConfiguration.DEFAULT_APPLICATION_TYPE; @@ -325,6 +326,11 @@ public long getStartTime() { } @Override + public long getSubmitTime() { + return start; + } + + @Override public long getLaunchTime() { return launch; } @@ -379,9 +385,9 @@ public ApplicationReport createAndGetApplicationReport( ApplicationReport report = ApplicationReport.newInstance( getApplicationId(), appAttemptId, getUser(), getQueue(), getName(), null, 0, null, null, getDiagnostics().toString(), - getTrackingUrl(), getLaunchTime(), getStartTime(), getFinishTime(), - getFinalApplicationStatus(), usageReport , null, getProgress(), - type, null); + getTrackingUrl(), getStartTime(), getSubmitTime(), getLaunchTime(), + getFinishTime(), getFinalApplicationStatus(), usageReport , null, + getProgress(), type, null); return report; } -- 2.7.4 (Apple Git-66)