diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ApplicationReport.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ApplicationReport.java index e5d7254..733561b 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ApplicationReport.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ApplicationReport.java @@ -60,6 +60,20 @@ public static ApplicationReport newInstance(ApplicationId applicationId, long startTime, long finishTime, FinalApplicationStatus finalStatus, ApplicationResourceUsageReport appResources, String origTrackingUrl, float progress, String applicationType, Token amRmToken) { + return newInstance(applicationId, applicationAttemptId, user, queue, name, + host, rpcPort, clientToAMToken, state, diagnostics, url, startTime, + finishTime, finalStatus, appResources, origTrackingUrl, progress, + applicationType, amRmToken, false); + } + + public static ApplicationReport newInstance(ApplicationId applicationId, + ApplicationAttemptId applicationAttemptId, String user, String queue, + String name, String host, int rpcPort, Token clientToAMToken, + YarnApplicationState state, String diagnostics, String url, + long startTime, long finishTime, FinalApplicationStatus finalStatus, + ApplicationResourceUsageReport appResources, String origTrackingUrl, + float progress, String applicationType, Token amRmToken, + boolean isUnamanagedAM) { ApplicationReport report = Records.newRecord(ApplicationReport.class); report.setApplicationId(applicationId); report.setCurrentApplicationAttemptId(applicationAttemptId); @@ -80,6 +94,7 @@ public static ApplicationReport newInstance(ApplicationId applicationId, report.setProgress(progress); report.setApplicationType(applicationType); report.setAMRMToken(amRmToken); + report.setUnmanagedAM(isUnamanagedAM); return report; } @@ -374,4 +389,18 @@ public static ApplicationReport newInstance(ApplicationId applicationId, @Unstable public abstract void setLogAggregationStatus( LogAggregationStatus logAggregationStatus); + + /** + * @return true if the AM is not managed by the RM + */ + @Public + @Unstable + public abstract boolean getUnmanagedAM(); + + /** + * @param value true if RM should not manage the AM + */ + @Public + @Unstable + public abstract void setUnmanagedAM(boolean value); } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/server/application_history_server.proto b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/server/application_history_server.proto index 0fcf2ac..1afff1c 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/server/application_history_server.proto +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/server/application_history_server.proto @@ -46,6 +46,7 @@ message ApplicationStartDataProto { optional string queue = 5; optional int64 submit_time = 6; optional int64 start_time = 7; + optional bool unmanaged_am = 8 [default = false]; } message ApplicationFinishDataProto { diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/yarn_protos.proto b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/yarn_protos.proto index c45081a..c05ab50 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/yarn_protos.proto +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/yarn_protos.proto @@ -195,6 +195,7 @@ message ApplicationReportProto { optional hadoop.common.TokenProto am_rm_token = 19; repeated string applicationTags = 20; optional LogAggregationStatusProto log_aggregation_status = 21; + optional bool unmanaged_am = 22 [default = false]; } enum LogAggregationStatusProto { diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/cli/ApplicationCLI.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/cli/ApplicationCLI.java index 8ef88c3..24deeeb 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/cli/ApplicationCLI.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/cli/ApplicationCLI.java @@ -534,7 +534,9 @@ private int printApplicationReport(String applicationId) appReportStr.println(appReport.getLogAggregationStatus() == null ? "N/A" : appReport.getLogAggregationStatus()); appReportStr.print("\tDiagnostics : "); - appReportStr.print(appReport.getDiagnostics()); + appReportStr.println(appReport.getDiagnostics()); + appReportStr.print("\tUnManagedAM : "); + appReportStr.print(appReport.getUnmanagedAM()); } else { appReportStr.print("Application with id '" + applicationId + "' doesn't exist in RM."); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/api/impl/TestYarnClient.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/api/impl/TestYarnClient.java index 10b9bbb..3d4b649 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/api/impl/TestYarnClient.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/api/impl/TestYarnClient.java @@ -845,12 +845,12 @@ public void testAMMRTokens() throws Exception { rmClient.start(); ApplicationId appId = createApp(rmClient, false); - waitTillAccepted(rmClient, appId); + waitTillAccepted(rmClient, appId, false); //managed AMs don't return AMRM token Assert.assertNull(rmClient.getAMRMToken(appId)); appId = createApp(rmClient, true); - waitTillAccepted(rmClient, appId); + waitTillAccepted(rmClient, appId, true); long start = System.currentTimeMillis(); while (rmClient.getAMRMToken(appId) == null) { if (System.currentTimeMillis() - start > 20 * 1000) { @@ -871,7 +871,7 @@ public ApplicationId run() throws Exception { rmClient.init(yarnConf); rmClient.start(); ApplicationId appId = createApp(rmClient, true); - waitTillAccepted(rmClient, appId); + waitTillAccepted(rmClient, appId, true); long start = System.currentTimeMillis(); while (rmClient.getAMRMToken(appId) == null) { if (System.currentTimeMillis() - start > 20 * 1000) { @@ -931,7 +931,8 @@ private ApplicationId createApp(YarnClient rmClient, boolean unmanaged) return appId; } - private void waitTillAccepted(YarnClient rmClient, ApplicationId appId) + private void waitTillAccepted(YarnClient rmClient, ApplicationId appId, + boolean isUnManagedAM) throws Exception { try { long start = System.currentTimeMillis(); @@ -944,6 +945,7 @@ private void waitTillAccepted(YarnClient rmClient, ApplicationId appId) Thread.sleep(200); report = rmClient.getApplicationReport(appId); } + Assert.assertEquals(isUnManagedAM, report.getUnmanagedAM()); } catch (Exception ex) { throw new Exception(ex); } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/cli/TestYarnCLI.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/cli/TestYarnCLI.java index 1013958..01ef93e 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/cli/TestYarnCLI.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/cli/TestYarnCLI.java @@ -106,6 +106,7 @@ public void testGetApplicationReport() throws Exception { FinalApplicationStatus.SUCCEEDED, usageReport, "N/A", 0.53789f, "YARN", null); newApplicationReport.setLogAggregationStatus(LogAggregationStatus.SUCCEEDED); + newApplicationReport.setUnmanagedAM(false); when(client.getApplicationReport(any(ApplicationId.class))).thenReturn( newApplicationReport); int result = cli.run(new String[] { "application", "-status", applicationId.toString() }); @@ -131,6 +132,7 @@ public void testGetApplicationReport() throws Exception { (i == 0 ? "N/A" : "123456 MB-seconds, 4567 vcore-seconds")); pw.println("\tLog Aggregation Status : SUCCEEDED"); pw.println("\tDiagnostics : diagnostics"); + pw.println("\tUnManagedAM : false"); pw.close(); String appReportStr = baos.toString("UTF-8"); Assert.assertEquals(appReportStr, sysOutStream.toString()); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ApplicationReportPBImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ApplicationReportPBImpl.java index 751dd90..d8befb6 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ApplicationReportPBImpl.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ApplicationReportPBImpl.java @@ -581,4 +581,16 @@ private LogAggregationStatus convertFromProtoFormat( convertToProtoFormat(LogAggregationStatus s) { return ProtoUtils.convertToProtoFormat(s); } + + @Override + public boolean getUnmanagedAM() { + ApplicationReportProtoOrBuilder p = viaProto ? proto : builder; + return p.getUnmanagedAm(); + } + + @Override + public void setUnmanagedAM(boolean value) { + maybeInitBuilder(); + builder.setUnmanagedAm(value); + } } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/ApplicationHistoryManagerImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/ApplicationHistoryManagerImpl.java index c7cf07b..f7a8450 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/ApplicationHistoryManagerImpl.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/ApplicationHistoryManagerImpl.java @@ -139,7 +139,7 @@ private ApplicationReport convertToApplicationReport( appHistory.getYarnApplicationState(), appHistory.getDiagnosticsInfo(), trackingUrl, appHistory.getStartTime(), appHistory.getFinishTime(), appHistory.getFinalApplicationStatus(), null, "", 100, - appHistory.getApplicationType(), null); + appHistory.getApplicationType(), null, appHistory.getUnmanagedAM()); } private ApplicationAttemptHistoryData getLastAttempt(ApplicationId appId) diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/ApplicationHistoryManagerOnTimelineStore.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/ApplicationHistoryManagerOnTimelineStore.java index db00d2c..b207dcd 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/ApplicationHistoryManagerOnTimelineStore.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/ApplicationHistoryManagerOnTimelineStore.java @@ -217,6 +217,7 @@ private static ApplicationReportExt convertToApplicationReport( String queue = null; String name = null; String type = null; + boolean isUnmanagedAM = false; long createdTime = 0; long finishedTime = 0; float progress = 0.0f; @@ -263,6 +264,13 @@ private static ApplicationReportExt convertToApplicationReport( entityInfo.get(ApplicationMetricsConstants.TYPE_ENTITY_INFO) .toString(); } + if (entityInfo + .containsKey(ApplicationMetricsConstants.UNMANAGED_AM_ENTITY_INFO)) { + isUnmanagedAM = + Boolean.parseBoolean(entityInfo.get( + ApplicationMetricsConstants.UNMANAGED_AM_ENTITY_INFO) + .toString()); + } if (entityInfo.containsKey(ApplicationMetricsConstants.APP_CPU_METRICS)) { long vcoreSeconds=Long.parseLong(entityInfo.get( ApplicationMetricsConstants.APP_CPU_METRICS).toString()); @@ -323,7 +331,7 @@ private static ApplicationReportExt convertToApplicationReport( ConverterUtils.toApplicationId(entity.getEntityId()), latestApplicationAttemptId, user, queue, name, null, -1, null, state, diagnosticsInfo, null, createdTime, finishedTime, finalStatus, appResources, - null, progress, type, null), appViewACLs); + null, progress, type, null, isUnmanagedAM), appViewACLs); } private static ApplicationAttemptReport convertToApplicationAttemptReport( diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/records/ApplicationHistoryData.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/records/ApplicationHistoryData.java index b7d16f3..4b25488 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/records/ApplicationHistoryData.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/records/ApplicationHistoryData.java @@ -54,6 +54,8 @@ private YarnApplicationState yarnApplicationState; + private boolean unManagedAM; + @Public @Unstable public static ApplicationHistoryData newInstance(ApplicationId applicationId, @@ -210,4 +212,22 @@ public YarnApplicationState getYarnApplicationState() { this.yarnApplicationState = yarnApplicationState; } + /** + * @return true if the AM is not managed by the RM + */ + @Public + @Unstable + public boolean getUnmanagedAM() { + return unManagedAM; + } + + /** + * @param value true if RM should not manage the AM + */ + @Public + @Unstable + public void setUnmanagedAM(boolean unManagedAM) { + this.unManagedAM = unManagedAM; + } + } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/records/ApplicationStartData.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/records/ApplicationStartData.java index 6bc1323..a20ed34 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/records/ApplicationStartData.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/records/ApplicationStartData.java @@ -35,7 +35,7 @@ @Unstable public static ApplicationStartData newInstance(ApplicationId applicationId, String applicationName, String applicationType, String queue, - String user, long submitTime, long startTime) { + String user, long submitTime, long startTime, boolean isUnmanagedAM) { ApplicationStartData appSD = Records.newRecord(ApplicationStartData.class); appSD.setApplicationId(applicationId); appSD.setApplicationName(applicationName); @@ -44,6 +44,7 @@ public static ApplicationStartData newInstance(ApplicationId applicationId, appSD.setUser(user); appSD.setSubmitTime(submitTime); appSD.setStartTime(startTime); + appSD.setUnmanagedAM(isUnmanagedAM); return appSD; } @@ -103,4 +104,12 @@ public static ApplicationStartData newInstance(ApplicationId applicationId, @Unstable public abstract void setStartTime(long startTime); + @Public + @Unstable + public abstract boolean getUnmanagedAM(); + + @Public + @Unstable + public abstract void setUnmanagedAM(boolean value); + } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/records/impl/pb/ApplicationStartDataPBImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/records/impl/pb/ApplicationStartDataPBImpl.java index 56f7aff..838b37c 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/records/impl/pb/ApplicationStartDataPBImpl.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/records/impl/pb/ApplicationStartDataPBImpl.java @@ -23,6 +23,7 @@ import org.apache.hadoop.yarn.proto.ApplicationHistoryServerProtos.ApplicationStartDataProto; import org.apache.hadoop.yarn.proto.ApplicationHistoryServerProtos.ApplicationStartDataProtoOrBuilder; import org.apache.hadoop.yarn.proto.YarnProtos.ApplicationIdProto; +import org.apache.hadoop.yarn.proto.YarnProtos.ApplicationReportProtoOrBuilder; import org.apache.hadoop.yarn.server.applicationhistoryservice.records.ApplicationStartData; import com.google.protobuf.TextFormat; @@ -226,4 +227,16 @@ private ApplicationIdPBImpl convertFromProtoFormat( ApplicationIdProto applicationId) { return new ApplicationIdPBImpl(applicationId); } + + @Override + public boolean getUnmanagedAM() { + ApplicationStartDataProtoOrBuilder p = viaProto ? proto : builder; + return p.getUnmanagedAm(); + } + + @Override + public void setUnmanagedAM(boolean value) { + maybeInitBuilder(); + builder.setUnmanagedAm(value); + } } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/ApplicationHistoryStoreTestUtils.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/ApplicationHistoryStoreTestUtils.java index de4051a..ad1b8ac 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/ApplicationHistoryStoreTestUtils.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/ApplicationHistoryStoreTestUtils.java @@ -44,7 +44,7 @@ protected void writeApplicationStartData(ApplicationId appId) throws IOException { store.applicationStarted(ApplicationStartData.newInstance(appId, - appId.toString(), "test type", "test queue", "test user", 0, 0)); + appId.toString(), "test type", "test queue", "test user", 0, 0, false)); } protected void writeApplicationFinishData(ApplicationId appId) diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/TestApplicationHistoryManagerOnTimelineStore.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/TestApplicationHistoryManagerOnTimelineStore.java index 8cf1240..2d3d16b 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/TestApplicationHistoryManagerOnTimelineStore.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/TestApplicationHistoryManagerOnTimelineStore.java @@ -446,6 +446,7 @@ private static TimelineEntity createApplicationTimelineEntity( "test app type"); entityInfo.put(ApplicationMetricsConstants.USER_ENTITY_INFO, "user1"); entityInfo.put(ApplicationMetricsConstants.QUEUE_ENTITY_INFO, "test queue"); + entityInfo.put(ApplicationMetricsConstants.UNMANAGED_AM_ENTITY_INFO, "false"); entityInfo.put(ApplicationMetricsConstants.SUBMITTED_TIME_ENTITY_INFO, Integer.MAX_VALUE + 1L); entityInfo.put(ApplicationMetricsConstants.APP_MEM_METRICS,123); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/metrics/ApplicationMetricsConstants.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/metrics/ApplicationMetricsConstants.java index df8eecb..8fd289a 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/metrics/ApplicationMetricsConstants.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/metrics/ApplicationMetricsConstants.java @@ -73,4 +73,6 @@ public static final String LATEST_APP_ATTEMPT_EVENT_INFO = "YARN_APPLICATION_LATEST_APP_ATTEMPT"; + public static final String UNMANAGED_AM_ENTITY_INFO = + "YARN_APPLICATION_UNMANAGED_AM"; } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/utils/BuilderUtils.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/utils/BuilderUtils.java index f2146c8..dea0ade 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/utils/BuilderUtils.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/utils/BuilderUtils.java @@ -324,7 +324,8 @@ public static ApplicationReport newApplicationReport( String url, long startTime, long finishTime, FinalApplicationStatus finalStatus, ApplicationResourceUsageReport appResources, String origTrackingUrl, - float progress, String appType, Token amRmToken, Set tags) { + float progress, String appType, Token amRmToken, Set tags, + boolean isUnmanagedAM) { ApplicationReport report = recordFactory .newRecordInstance(ApplicationReport.class); report.setApplicationId(applicationId); @@ -347,6 +348,7 @@ public static ApplicationReport newApplicationReport( report.setApplicationType(appType); report.setAMRMToken(amRmToken); report.setApplicationTags(tags); + report.setUnmanagedAM(isUnmanagedAM); return report; } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/webapp/AppBlock.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/webapp/AppBlock.java index dd5a4c8..fd0285b 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/webapp/AppBlock.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/webapp/AppBlock.java @@ -197,6 +197,7 @@ public ApplicationReport run() throws Exception { } overviewTable._("Diagnostics:", app.getDiagnosticsInfo() == null ? "" : app.getDiagnosticsInfo()); + overviewTable._("UnManaged AM:", app.getUnmanagedAM()); Collection attempts; try { diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/webapp/dao/AppInfo.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/webapp/dao/AppInfo.java index 3eafb8c..6a7c6f6 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/webapp/dao/AppInfo.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/webapp/dao/AppInfo.java @@ -53,6 +53,7 @@ protected long finishedTime; protected long elapsedTime; protected String applicationTags; + protected boolean unManagedAM; public AppInfo() { // JAXB needs this @@ -86,6 +87,7 @@ public AppInfo(ApplicationReport app) { if (app.getApplicationTags() != null && !app.getApplicationTags().isEmpty()) { this.applicationTags = CSV_JOINER.join(app.getApplicationTags()); } + unManagedAM = app.getUnmanagedAM(); } public String getAppId() { @@ -167,4 +169,8 @@ public long getElapsedTime() { public String getApplicationTags() { return applicationTags; } + + public boolean getUnmanagedAM() { + return unManagedAM; + } } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ahs/RMApplicationHistoryWriter.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ahs/RMApplicationHistoryWriter.java index bd328ab..45adac3 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ahs/RMApplicationHistoryWriter.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ahs/RMApplicationHistoryWriter.java @@ -228,7 +228,8 @@ public void applicationStarted(RMApp app) { new WritingApplicationStartEvent(app.getApplicationId(), ApplicationStartData.newInstance(app.getApplicationId(), app.getName(), app.getApplicationType(), app.getQueue(), app.getUser(), - app.getSubmitTime(), app.getStartTime()))); + app.getSubmitTime(), app.getStartTime(), + app.getApplicationSubmissionContext().getUnmanagedAM()))); } } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/metrics/ApplicationCreatedEvent.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/metrics/ApplicationCreatedEvent.java index 2373b3b..d800845 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/metrics/ApplicationCreatedEvent.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/metrics/ApplicationCreatedEvent.java @@ -29,6 +29,7 @@ private String user; private String queue; private long submittedTime; + private boolean unmanaged_am; public ApplicationCreatedEvent(ApplicationId appId, String name, @@ -36,7 +37,8 @@ public ApplicationCreatedEvent(ApplicationId appId, String user, String queue, long submittedTime, - long createdTime) { + long createdTime, + boolean isUnmanagedAM) { super(SystemMetricsEventType.APP_CREATED, createdTime); this.appId = appId; this.name = name; @@ -44,6 +46,7 @@ public ApplicationCreatedEvent(ApplicationId appId, this.user = user; this.queue = queue; this.submittedTime = submittedTime; + this.unmanaged_am = isUnmanagedAM; } @Override @@ -75,4 +78,8 @@ public long getSubmittedTime() { return submittedTime; } + public boolean getUnmanagedAM() { + return unmanaged_am; + } + } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/metrics/SystemMetricsPublisher.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/metrics/SystemMetricsPublisher.java index b849b00..230c14e 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/metrics/SystemMetricsPublisher.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/metrics/SystemMetricsPublisher.java @@ -106,7 +106,8 @@ public void appCreated(RMApp app, long createdTime) { app.getUser(), app.getQueue(), app.getSubmitTime(), - createdTime)); + createdTime, + app.getApplicationSubmissionContext().getUnmanagedAM())); } } @@ -251,6 +252,8 @@ private void publishApplicationCreatedEvent(ApplicationCreatedEvent event) { event.getQueue()); entityInfo.put(ApplicationMetricsConstants.SUBMITTED_TIME_ENTITY_INFO, event.getSubmittedTime()); + entityInfo.put(ApplicationMetricsConstants.UNMANAGED_AM_ENTITY_INFO, + event.getUnmanagedAM()); entity.setOtherInfo(entityInfo); TimelineEvent tEvent = new TimelineEvent(); tEvent.setEventType( diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/RMAppImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/RMAppImpl.java index 8abc478..d492ea8 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/RMAppImpl.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/RMAppImpl.java @@ -644,7 +644,7 @@ public ApplicationReport createAndGetApplicationReport(String clientUserName, createApplicationState(), diags, trackingUrl, this.startTime, this.finishTime, finishState, appUsageReport, origTrackingUrl, progress, this.applicationType, - amrmToken, applicationTags); + amrmToken, applicationTags,submissionContext.getUnmanagedAM()); report.setLogAggregationStatus(logAggregationStatus); return report; } finally { diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/AppInfo.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/AppInfo.java index bd3b046..d728269 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/AppInfo.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/AppInfo.java @@ -96,6 +96,7 @@ protected List resourceRequests; protected LogAggregationStatus logAggregationStatus; + protected boolean unManagedAM; public AppInfo() { } // JAXB needs this @@ -183,6 +184,7 @@ public AppInfo(ResourceManager rm, RMApp app, Boolean hasAccess, appMetrics.getResourcePreempted().getVirtualCores(); memorySeconds = appMetrics.getMemorySeconds(); vcoreSeconds = appMetrics.getVcoreSeconds(); + unManagedAM = app.getApplicationSubmissionContext().getUnmanagedAM(); } } @@ -321,4 +323,8 @@ public long getVcoreSeconds() { public LogAggregationStatus getLogAggregationStatus() { return this.logAggregationStatus; } + + public boolean getUnmanagedAM() { + return unManagedAM; + } } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/ahs/TestRMApplicationHistoryWriter.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/ahs/TestRMApplicationHistoryWriter.java index f827bf4..985b3b6 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/ahs/TestRMApplicationHistoryWriter.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/ahs/TestRMApplicationHistoryWriter.java @@ -32,6 +32,7 @@ import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.yarn.api.records.ApplicationAttemptId; import org.apache.hadoop.yarn.api.records.ApplicationId; +import org.apache.hadoop.yarn.api.records.ApplicationSubmissionContext; import org.apache.hadoop.yarn.api.records.Container; import org.apache.hadoop.yarn.api.records.ContainerId; import org.apache.hadoop.yarn.api.records.ContainerState; @@ -142,6 +143,9 @@ private static RMApp createRMApp(ApplicationId appId) { new StringBuilder("test diagnostics info")); when(app.getFinalApplicationStatus()).thenReturn( FinalApplicationStatus.UNDEFINED); + ApplicationSubmissionContext asc = mock(ApplicationSubmissionContext.class); + when(asc.getUnmanagedAM()).thenReturn(false); + when(app.getApplicationSubmissionContext()).thenReturn(asc); return app; } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/metrics/TestSystemMetricsPublisher.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/metrics/TestSystemMetricsPublisher.java index 7ed3835..5b9befd 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/metrics/TestSystemMetricsPublisher.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/metrics/TestSystemMetricsPublisher.java @@ -25,6 +25,7 @@ import org.apache.hadoop.yarn.api.records.ApplicationAttemptId; import org.apache.hadoop.yarn.api.records.ApplicationId; +import org.apache.hadoop.yarn.api.records.ApplicationSubmissionContext; import org.apache.hadoop.yarn.api.records.Container; import org.apache.hadoop.yarn.api.records.ContainerId; import org.apache.hadoop.yarn.api.records.ContainerState; @@ -130,6 +131,11 @@ public void testPublishApplicationMetrics() throws Exception { Assert.assertEquals(app.getQueue(), entity.getOtherInfo() .get(ApplicationMetricsConstants.QUEUE_ENTITY_INFO)); + + Assert.assertEquals( + app.getApplicationSubmissionContext().getUnmanagedAM(), + entity.getOtherInfo().get( + ApplicationMetricsConstants.UNMANAGED_AM_ENTITY_INFO)); Assert .assertEquals( app.getUser(), @@ -352,6 +358,9 @@ private static RMApp createRMApp(ApplicationId appId) { FinalApplicationStatus.UNDEFINED); when(app.getRMAppMetrics()).thenReturn( new RMAppMetrics(null, 0, 0, Integer.MAX_VALUE, Long.MAX_VALUE)); + ApplicationSubmissionContext asc = mock(ApplicationSubmissionContext.class); + when(asc.getUnmanagedAM()).thenReturn(false); + when(app.getApplicationSubmissionContext()).thenReturn(asc); return app; } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesApps.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesApps.java index 549b9e0..4e0f51b 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesApps.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesApps.java @@ -1315,7 +1315,7 @@ public void verifyAppsXML(NodeList nodes, RMApp app) throws JSONException, public void verifyAppInfo(JSONObject info, RMApp app) throws JSONException, Exception { - assertEquals("incorrect number of elements", 28, info.length()); + assertEquals("incorrect number of elements", 29, info.length()); verifyAppInfoGeneric(app, info.getString("id"), info.getString("user"), info.getString("name"), info.getString("applicationType"), -- 1.9.2.msysgit.0