diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ApplicationReport.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ApplicationReport.java index e5d7254..9204fb9 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ApplicationReport.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ApplicationReport.java @@ -374,4 +374,18 @@ public static ApplicationReport newInstance(ApplicationId applicationId, @Unstable public abstract void setLogAggregationStatus( LogAggregationStatus logAggregationStatus); + + /** + * @return true if the AM is not managed by the RM + */ + @Public + @Unstable + public abstract boolean isUnmanagedApp(); + + /** + * @param value true if RM should not manage the AM + */ + @Public + @Unstable + public abstract void setUnmanagedApp(boolean unmanagedApplication); } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/server/application_history_server.proto b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/server/application_history_server.proto index 0fcf2ac..eed7038 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/server/application_history_server.proto +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/server/application_history_server.proto @@ -46,6 +46,7 @@ message ApplicationStartDataProto { optional string queue = 5; optional int64 submit_time = 6; optional int64 start_time = 7; + optional bool unmanaged_application = 8 [default = false]; } message ApplicationFinishDataProto { diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/yarn_protos.proto b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/yarn_protos.proto index c45081a..3e9c7b4 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/yarn_protos.proto +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/yarn_protos.proto @@ -195,6 +195,7 @@ message ApplicationReportProto { optional hadoop.common.TokenProto am_rm_token = 19; repeated string applicationTags = 20; optional LogAggregationStatusProto log_aggregation_status = 21; + optional bool unmanaged_application = 22 [default = false]; } enum LogAggregationStatusProto { diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/cli/ApplicationCLI.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/cli/ApplicationCLI.java index 8ef88c3..013b88e 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/cli/ApplicationCLI.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/cli/ApplicationCLI.java @@ -534,7 +534,9 @@ private int printApplicationReport(String applicationId) appReportStr.println(appReport.getLogAggregationStatus() == null ? "N/A" : appReport.getLogAggregationStatus()); appReportStr.print("\tDiagnostics : "); - appReportStr.print(appReport.getDiagnostics()); + appReportStr.println(appReport.getDiagnostics()); + appReportStr.print("\tUnmanaged Application : "); + appReportStr.print(appReport.isUnmanagedApp()); } else { appReportStr.print("Application with id '" + applicationId + "' doesn't exist in RM."); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/api/impl/TestYarnClient.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/api/impl/TestYarnClient.java index 10b9bbb..4b9be6f 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/api/impl/TestYarnClient.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/api/impl/TestYarnClient.java @@ -845,12 +845,12 @@ public void testAMMRTokens() throws Exception { rmClient.start(); ApplicationId appId = createApp(rmClient, false); - waitTillAccepted(rmClient, appId); + waitTillAccepted(rmClient, appId, false); //managed AMs don't return AMRM token Assert.assertNull(rmClient.getAMRMToken(appId)); appId = createApp(rmClient, true); - waitTillAccepted(rmClient, appId); + waitTillAccepted(rmClient, appId, true); long start = System.currentTimeMillis(); while (rmClient.getAMRMToken(appId) == null) { if (System.currentTimeMillis() - start > 20 * 1000) { @@ -871,7 +871,7 @@ public ApplicationId run() throws Exception { rmClient.init(yarnConf); rmClient.start(); ApplicationId appId = createApp(rmClient, true); - waitTillAccepted(rmClient, appId); + waitTillAccepted(rmClient, appId, true); long start = System.currentTimeMillis(); while (rmClient.getAMRMToken(appId) == null) { if (System.currentTimeMillis() - start > 20 * 1000) { @@ -931,7 +931,8 @@ private ApplicationId createApp(YarnClient rmClient, boolean unmanaged) return appId; } - private void waitTillAccepted(YarnClient rmClient, ApplicationId appId) + private void waitTillAccepted(YarnClient rmClient, ApplicationId appId, + boolean unmanagedApplication) throws Exception { try { long start = System.currentTimeMillis(); @@ -944,6 +945,7 @@ private void waitTillAccepted(YarnClient rmClient, ApplicationId appId) Thread.sleep(200); report = rmClient.getApplicationReport(appId); } + Assert.assertEquals(unmanagedApplication, report.isUnmanagedApp()); } catch (Exception ex) { throw new Exception(ex); } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/cli/TestYarnCLI.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/cli/TestYarnCLI.java index 1013958..4245fe4 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/cli/TestYarnCLI.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/cli/TestYarnCLI.java @@ -106,6 +106,7 @@ public void testGetApplicationReport() throws Exception { FinalApplicationStatus.SUCCEEDED, usageReport, "N/A", 0.53789f, "YARN", null); newApplicationReport.setLogAggregationStatus(LogAggregationStatus.SUCCEEDED); + newApplicationReport.setUnmanagedApp(false); when(client.getApplicationReport(any(ApplicationId.class))).thenReturn( newApplicationReport); int result = cli.run(new String[] { "application", "-status", applicationId.toString() }); @@ -131,6 +132,7 @@ public void testGetApplicationReport() throws Exception { (i == 0 ? "N/A" : "123456 MB-seconds, 4567 vcore-seconds")); pw.println("\tLog Aggregation Status : SUCCEEDED"); pw.println("\tDiagnostics : diagnostics"); + pw.println("\tUnmanaged Application : false"); pw.close(); String appReportStr = baos.toString("UTF-8"); Assert.assertEquals(appReportStr, sysOutStream.toString()); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ApplicationReportPBImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ApplicationReportPBImpl.java index 751dd90..69435b5 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ApplicationReportPBImpl.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ApplicationReportPBImpl.java @@ -581,4 +581,16 @@ private LogAggregationStatus convertFromProtoFormat( convertToProtoFormat(LogAggregationStatus s) { return ProtoUtils.convertToProtoFormat(s); } + + @Override + public boolean isUnmanagedApp() { + ApplicationReportProtoOrBuilder p = viaProto ? proto : builder; + return p.getUnmanagedApplication(); + } + + @Override + public void setUnmanagedApp(boolean unmanagedApplication) { + maybeInitBuilder(); + builder.setUnmanagedApplication(unmanagedApplication); + } } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/ApplicationHistoryManagerImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/ApplicationHistoryManagerImpl.java index c7cf07b..7bdb3b0 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/ApplicationHistoryManagerImpl.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/ApplicationHistoryManagerImpl.java @@ -94,7 +94,7 @@ public ContainerReport getAMContainer(ApplicationAttemptId appAttemptId) ApplicationReport app = getApplication(appAttemptId.getApplicationId()); return convertToContainerReport(historyStore.getAMContainer(appAttemptId), - app == null ? null : app.getUser()); + app.getUser()); } @Override @@ -133,13 +133,17 @@ private ApplicationReport convertToApplicationReport( host = lastAttempt.getHost(); rpcPort = lastAttempt.getRPCPort(); } - return ApplicationReport.newInstance(appHistory.getApplicationId(), - currentApplicationAttemptId, appHistory.getUser(), appHistory.getQueue(), - appHistory.getApplicationName(), host, rpcPort, null, - appHistory.getYarnApplicationState(), appHistory.getDiagnosticsInfo(), - trackingUrl, appHistory.getStartTime(), appHistory.getFinishTime(), - appHistory.getFinalApplicationStatus(), null, "", 100, - appHistory.getApplicationType(), null); + ApplicationReport appReport = + ApplicationReport.newInstance(appHistory.getApplicationId(), + currentApplicationAttemptId, appHistory.getUser(), + appHistory.getQueue(), appHistory.getApplicationName(), host, + rpcPort, null, appHistory.getYarnApplicationState(), + appHistory.getDiagnosticsInfo(), trackingUrl, + appHistory.getStartTime(), appHistory.getFinishTime(), + appHistory.getFinalApplicationStatus(), null, "", 100, + appHistory.getApplicationType(), null); + appReport.setUnmanagedApp(appHistory.isUnmanagedApp()); + return appReport; } private ApplicationAttemptHistoryData getLastAttempt(ApplicationId appId) @@ -197,7 +201,7 @@ public ContainerReport getContainer(ContainerId containerId) ApplicationReport app = getApplication(containerId.getApplicationAttemptId().getApplicationId()); return convertToContainerReport(historyStore.getContainer(containerId), - app == null ? null: app.getUser()); + app.getUser()); } private ContainerReport convertToContainerReport( @@ -229,8 +233,7 @@ private ContainerReport convertToContainerReport( new HashMap(); for (Entry entry : histData.entrySet()) { containersReport.put(entry.getKey(), - convertToContainerReport(entry.getValue(), - app == null ? null : app.getUser())); + convertToContainerReport(entry.getValue(), app.getUser())); } return containersReport; } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/ApplicationHistoryManagerOnTimelineStore.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/ApplicationHistoryManagerOnTimelineStore.java index db00d2c..745c047 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/ApplicationHistoryManagerOnTimelineStore.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/ApplicationHistoryManagerOnTimelineStore.java @@ -217,6 +217,7 @@ private static ApplicationReportExt convertToApplicationReport( String queue = null; String name = null; String type = null; + boolean unmanagedApplication = false; long createdTime = 0; long finishedTime = 0; float progress = 0.0f; @@ -263,6 +264,13 @@ private static ApplicationReportExt convertToApplicationReport( entityInfo.get(ApplicationMetricsConstants.TYPE_ENTITY_INFO) .toString(); } + if (entityInfo + .containsKey(ApplicationMetricsConstants.UNMANAGED_APPLICATION_ENTITY_INFO)) { + unmanagedApplication = + Boolean.parseBoolean(entityInfo.get( + ApplicationMetricsConstants.UNMANAGED_APPLICATION_ENTITY_INFO) + .toString()); + } if (entityInfo.containsKey(ApplicationMetricsConstants.APP_CPU_METRICS)) { long vcoreSeconds=Long.parseLong(entityInfo.get( ApplicationMetricsConstants.APP_CPU_METRICS).toString()); @@ -319,11 +327,14 @@ private static ApplicationReportExt convertToApplicationReport( } } } - return new ApplicationReportExt(ApplicationReport.newInstance( - ConverterUtils.toApplicationId(entity.getEntityId()), - latestApplicationAttemptId, user, queue, name, null, -1, null, state, - diagnosticsInfo, null, createdTime, finishedTime, finalStatus, appResources, - null, progress, type, null), appViewACLs); + ApplicationReport appReport = + ApplicationReport.newInstance( + ConverterUtils.toApplicationId(entity.getEntityId()), + latestApplicationAttemptId, user, queue, name, null, -1, null, + state, diagnosticsInfo, null, createdTime, finishedTime, + finalStatus, appResources, null, progress, type, null); + appReport.setUnmanagedApp(unmanagedApplication); + return new ApplicationReportExt(appReport, appViewACLs); } private static ApplicationAttemptReport convertToApplicationAttemptReport( diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/records/ApplicationHistoryData.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/records/ApplicationHistoryData.java index b7d16f3..5e2796b 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/records/ApplicationHistoryData.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/records/ApplicationHistoryData.java @@ -54,6 +54,8 @@ private YarnApplicationState yarnApplicationState; + private boolean unmanagedApplication; + @Public @Unstable public static ApplicationHistoryData newInstance(ApplicationId applicationId, @@ -210,4 +212,22 @@ public YarnApplicationState getYarnApplicationState() { this.yarnApplicationState = yarnApplicationState; } + /** + * @return true if the AM is not managed by the RM + */ + @Public + @Unstable + public boolean isUnmanagedApp() { + return unmanagedApplication; + } + + /** + * @param value true if RM should not manage the AM + */ + @Public + @Unstable + public void setUnmanagedApp(boolean unmanagedApplication) { + this.unmanagedApplication = unmanagedApplication; + } + } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/records/ApplicationStartData.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/records/ApplicationStartData.java index 6bc1323..85d888b 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/records/ApplicationStartData.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/records/ApplicationStartData.java @@ -35,7 +35,7 @@ @Unstable public static ApplicationStartData newInstance(ApplicationId applicationId, String applicationName, String applicationType, String queue, - String user, long submitTime, long startTime) { + String user, long submitTime, long startTime, boolean isUnmanagedAM) { ApplicationStartData appSD = Records.newRecord(ApplicationStartData.class); appSD.setApplicationId(applicationId); appSD.setApplicationName(applicationName); @@ -44,6 +44,7 @@ public static ApplicationStartData newInstance(ApplicationId applicationId, appSD.setUser(user); appSD.setSubmitTime(submitTime); appSD.setStartTime(startTime); + appSD.setUnmanagedApp(isUnmanagedAM); return appSD; } @@ -103,4 +104,12 @@ public static ApplicationStartData newInstance(ApplicationId applicationId, @Unstable public abstract void setStartTime(long startTime); + @Public + @Unstable + public abstract boolean isUnmanagedApp(); + + @Public + @Unstable + public abstract void setUnmanagedApp(boolean unmanagedApplication); + } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/records/impl/pb/ApplicationStartDataPBImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/records/impl/pb/ApplicationStartDataPBImpl.java index 56f7aff..c8fdb3a 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/records/impl/pb/ApplicationStartDataPBImpl.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/records/impl/pb/ApplicationStartDataPBImpl.java @@ -23,6 +23,7 @@ import org.apache.hadoop.yarn.proto.ApplicationHistoryServerProtos.ApplicationStartDataProto; import org.apache.hadoop.yarn.proto.ApplicationHistoryServerProtos.ApplicationStartDataProtoOrBuilder; import org.apache.hadoop.yarn.proto.YarnProtos.ApplicationIdProto; +import org.apache.hadoop.yarn.proto.YarnProtos.ApplicationReportProtoOrBuilder; import org.apache.hadoop.yarn.server.applicationhistoryservice.records.ApplicationStartData; import com.google.protobuf.TextFormat; @@ -226,4 +227,16 @@ private ApplicationIdPBImpl convertFromProtoFormat( ApplicationIdProto applicationId) { return new ApplicationIdPBImpl(applicationId); } + + @Override + public boolean isUnmanagedApp() { + ApplicationStartDataProtoOrBuilder p = viaProto ? proto : builder; + return p.getUnmanagedApplication(); + } + + @Override + public void setUnmanagedApp(boolean unmanagedApplication) { + maybeInitBuilder(); + builder.setUnmanagedApplication(unmanagedApplication); + } } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/ApplicationHistoryStoreTestUtils.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/ApplicationHistoryStoreTestUtils.java index de4051a..ad1b8ac 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/ApplicationHistoryStoreTestUtils.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/ApplicationHistoryStoreTestUtils.java @@ -44,7 +44,7 @@ protected void writeApplicationStartData(ApplicationId appId) throws IOException { store.applicationStarted(ApplicationStartData.newInstance(appId, - appId.toString(), "test type", "test queue", "test user", 0, 0)); + appId.toString(), "test type", "test queue", "test user", 0, 0, false)); } protected void writeApplicationFinishData(ApplicationId appId) diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/TestApplicationHistoryManagerOnTimelineStore.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/TestApplicationHistoryManagerOnTimelineStore.java index 8cf1240..432fb91 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/TestApplicationHistoryManagerOnTimelineStore.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/TestApplicationHistoryManagerOnTimelineStore.java @@ -446,6 +446,8 @@ private static TimelineEntity createApplicationTimelineEntity( "test app type"); entityInfo.put(ApplicationMetricsConstants.USER_ENTITY_INFO, "user1"); entityInfo.put(ApplicationMetricsConstants.QUEUE_ENTITY_INFO, "test queue"); + entityInfo.put( + ApplicationMetricsConstants.UNMANAGED_APPLICATION_ENTITY_INFO, "false"); entityInfo.put(ApplicationMetricsConstants.SUBMITTED_TIME_ENTITY_INFO, Integer.MAX_VALUE + 1L); entityInfo.put(ApplicationMetricsConstants.APP_MEM_METRICS,123); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/metrics/ApplicationMetricsConstants.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/metrics/ApplicationMetricsConstants.java index df8eecb..19a5de6 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/metrics/ApplicationMetricsConstants.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/metrics/ApplicationMetricsConstants.java @@ -73,4 +73,6 @@ public static final String LATEST_APP_ATTEMPT_EVENT_INFO = "YARN_APPLICATION_LATEST_APP_ATTEMPT"; + public static final String UNMANAGED_APPLICATION_ENTITY_INFO = + "YARN_APPLICATION_UNMANAGED_APPLICATION"; } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/utils/BuilderUtils.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/utils/BuilderUtils.java index f2146c8..719ce99 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/utils/BuilderUtils.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/utils/BuilderUtils.java @@ -324,7 +324,8 @@ public static ApplicationReport newApplicationReport( String url, long startTime, long finishTime, FinalApplicationStatus finalStatus, ApplicationResourceUsageReport appResources, String origTrackingUrl, - float progress, String appType, Token amRmToken, Set tags) { + float progress, String appType, Token amRmToken, Set tags, + boolean unmanagedApplication) { ApplicationReport report = recordFactory .newRecordInstance(ApplicationReport.class); report.setApplicationId(applicationId); @@ -347,6 +348,7 @@ public static ApplicationReport newApplicationReport( report.setApplicationType(appType); report.setAMRMToken(amRmToken); report.setApplicationTags(tags); + report.setUnmanagedApp(unmanagedApplication); return report; } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/webapp/AppBlock.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/webapp/AppBlock.java index dd5a4c8..8fd2ae5 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/webapp/AppBlock.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/webapp/AppBlock.java @@ -197,6 +197,7 @@ public ApplicationReport run() throws Exception { } overviewTable._("Diagnostics:", app.getDiagnosticsInfo() == null ? "" : app.getDiagnosticsInfo()); + overviewTable._("Unmanaged Application:", app.getUnmanagedAM()); Collection attempts; try { diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/webapp/dao/AppInfo.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/webapp/dao/AppInfo.java index 3eafb8c..04dbcbc 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/webapp/dao/AppInfo.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/webapp/dao/AppInfo.java @@ -53,6 +53,7 @@ protected long finishedTime; protected long elapsedTime; protected String applicationTags; + protected boolean unmanagedApplication; public AppInfo() { // JAXB needs this @@ -86,6 +87,7 @@ public AppInfo(ApplicationReport app) { if (app.getApplicationTags() != null && !app.getApplicationTags().isEmpty()) { this.applicationTags = CSV_JOINER.join(app.getApplicationTags()); } + unmanagedApplication = app.isUnmanagedApp(); } public String getAppId() { @@ -167,4 +169,8 @@ public long getElapsedTime() { public String getApplicationTags() { return applicationTags; } + + public boolean getUnmanagedAM() { + return unmanagedApplication; + } } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ahs/RMApplicationHistoryWriter.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ahs/RMApplicationHistoryWriter.java index bd328ab..45adac3 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ahs/RMApplicationHistoryWriter.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ahs/RMApplicationHistoryWriter.java @@ -228,7 +228,8 @@ public void applicationStarted(RMApp app) { new WritingApplicationStartEvent(app.getApplicationId(), ApplicationStartData.newInstance(app.getApplicationId(), app.getName(), app.getApplicationType(), app.getQueue(), app.getUser(), - app.getSubmitTime(), app.getStartTime()))); + app.getSubmitTime(), app.getStartTime(), + app.getApplicationSubmissionContext().getUnmanagedAM()))); } } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/metrics/ApplicationCreatedEvent.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/metrics/ApplicationCreatedEvent.java index 2373b3b..228c17e 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/metrics/ApplicationCreatedEvent.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/metrics/ApplicationCreatedEvent.java @@ -29,6 +29,7 @@ private String user; private String queue; private long submittedTime; + private boolean unmanagedApplication; public ApplicationCreatedEvent(ApplicationId appId, String name, @@ -36,7 +37,8 @@ public ApplicationCreatedEvent(ApplicationId appId, String user, String queue, long submittedTime, - long createdTime) { + long createdTime, + boolean unmanagedApplication) { super(SystemMetricsEventType.APP_CREATED, createdTime); this.appId = appId; this.name = name; @@ -44,6 +46,7 @@ public ApplicationCreatedEvent(ApplicationId appId, this.user = user; this.queue = queue; this.submittedTime = submittedTime; + this.unmanagedApplication = unmanagedApplication; } @Override @@ -75,4 +78,8 @@ public long getSubmittedTime() { return submittedTime; } + public boolean isUnmanagedApp() { + return unmanagedApplication; + } + } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/metrics/SystemMetricsPublisher.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/metrics/SystemMetricsPublisher.java index b849b00..b0908ad 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/metrics/SystemMetricsPublisher.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/metrics/SystemMetricsPublisher.java @@ -106,7 +106,8 @@ public void appCreated(RMApp app, long createdTime) { app.getUser(), app.getQueue(), app.getSubmitTime(), - createdTime)); + createdTime, + app.getApplicationSubmissionContext().getUnmanagedAM())); } } @@ -251,6 +252,9 @@ private void publishApplicationCreatedEvent(ApplicationCreatedEvent event) { event.getQueue()); entityInfo.put(ApplicationMetricsConstants.SUBMITTED_TIME_ENTITY_INFO, event.getSubmittedTime()); + entityInfo.put( + ApplicationMetricsConstants.UNMANAGED_APPLICATION_ENTITY_INFO, + event.isUnmanagedApp()); entity.setOtherInfo(entityInfo); TimelineEvent tEvent = new TimelineEvent(); tEvent.setEventType( diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/RMAppImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/RMAppImpl.java index 8abc478..d492ea8 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/RMAppImpl.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/RMAppImpl.java @@ -644,7 +644,7 @@ public ApplicationReport createAndGetApplicationReport(String clientUserName, createApplicationState(), diags, trackingUrl, this.startTime, this.finishTime, finishState, appUsageReport, origTrackingUrl, progress, this.applicationType, - amrmToken, applicationTags); + amrmToken, applicationTags,submissionContext.getUnmanagedAM()); report.setLogAggregationStatus(logAggregationStatus); return report; } finally { diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/AppInfo.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/AppInfo.java index bd3b046..14bc73c 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/AppInfo.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/AppInfo.java @@ -96,6 +96,7 @@ protected List resourceRequests; protected LogAggregationStatus logAggregationStatus; + protected boolean unmanagedApplication; public AppInfo() { } // JAXB needs this @@ -183,6 +184,7 @@ public AppInfo(ResourceManager rm, RMApp app, Boolean hasAccess, appMetrics.getResourcePreempted().getVirtualCores(); memorySeconds = appMetrics.getMemorySeconds(); vcoreSeconds = appMetrics.getVcoreSeconds(); + unmanagedApplication = app.getApplicationSubmissionContext().getUnmanagedAM(); } } @@ -321,4 +323,8 @@ public long getVcoreSeconds() { public LogAggregationStatus getLogAggregationStatus() { return this.logAggregationStatus; } + + public boolean getUnmanagedAM() { + return unmanagedApplication; + } } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/ahs/TestRMApplicationHistoryWriter.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/ahs/TestRMApplicationHistoryWriter.java index f827bf4..985b3b6 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/ahs/TestRMApplicationHistoryWriter.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/ahs/TestRMApplicationHistoryWriter.java @@ -32,6 +32,7 @@ import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.yarn.api.records.ApplicationAttemptId; import org.apache.hadoop.yarn.api.records.ApplicationId; +import org.apache.hadoop.yarn.api.records.ApplicationSubmissionContext; import org.apache.hadoop.yarn.api.records.Container; import org.apache.hadoop.yarn.api.records.ContainerId; import org.apache.hadoop.yarn.api.records.ContainerState; @@ -142,6 +143,9 @@ private static RMApp createRMApp(ApplicationId appId) { new StringBuilder("test diagnostics info")); when(app.getFinalApplicationStatus()).thenReturn( FinalApplicationStatus.UNDEFINED); + ApplicationSubmissionContext asc = mock(ApplicationSubmissionContext.class); + when(asc.getUnmanagedAM()).thenReturn(false); + when(app.getApplicationSubmissionContext()).thenReturn(asc); return app; } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/metrics/TestSystemMetricsPublisher.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/metrics/TestSystemMetricsPublisher.java index 7ed3835..530648c 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/metrics/TestSystemMetricsPublisher.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/metrics/TestSystemMetricsPublisher.java @@ -25,6 +25,7 @@ import org.apache.hadoop.yarn.api.records.ApplicationAttemptId; import org.apache.hadoop.yarn.api.records.ApplicationId; +import org.apache.hadoop.yarn.api.records.ApplicationSubmissionContext; import org.apache.hadoop.yarn.api.records.Container; import org.apache.hadoop.yarn.api.records.ContainerId; import org.apache.hadoop.yarn.api.records.ContainerState; @@ -130,6 +131,11 @@ public void testPublishApplicationMetrics() throws Exception { Assert.assertEquals(app.getQueue(), entity.getOtherInfo() .get(ApplicationMetricsConstants.QUEUE_ENTITY_INFO)); + + Assert.assertEquals( + app.getApplicationSubmissionContext().getUnmanagedAM(), + entity.getOtherInfo().get( + ApplicationMetricsConstants.UNMANAGED_APPLICATION_ENTITY_INFO)); Assert .assertEquals( app.getUser(), @@ -352,6 +358,9 @@ private static RMApp createRMApp(ApplicationId appId) { FinalApplicationStatus.UNDEFINED); when(app.getRMAppMetrics()).thenReturn( new RMAppMetrics(null, 0, 0, Integer.MAX_VALUE, Long.MAX_VALUE)); + ApplicationSubmissionContext asc = mock(ApplicationSubmissionContext.class); + when(asc.getUnmanagedAM()).thenReturn(false); + when(app.getApplicationSubmissionContext()).thenReturn(asc); return app; } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesApps.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesApps.java index 549b9e0..879ce4c 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesApps.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesApps.java @@ -1308,14 +1308,15 @@ public void verifyAppsXML(NodeList nodes, RMApp app) throws JSONException, WebServicesTestUtils.getXmlInt(element, "preemptedResourceVCores"), WebServicesTestUtils.getXmlInt(element, "numNonAMContainerPreempted"), WebServicesTestUtils.getXmlInt(element, "numAMContainerPreempted"), - WebServicesTestUtils.getXmlString(element, "logAggregationStatus")); + WebServicesTestUtils.getXmlString(element, "logAggregationStatus"), + WebServicesTestUtils.getXmlBoolean(element, "unmanagedApplication")); } } public void verifyAppInfo(JSONObject info, RMApp app) throws JSONException, Exception { - assertEquals("incorrect number of elements", 28, info.length()); + assertEquals("incorrect number of elements", 29, info.length()); verifyAppInfoGeneric(app, info.getString("id"), info.getString("user"), info.getString("name"), info.getString("applicationType"), @@ -1331,7 +1332,8 @@ public void verifyAppInfo(JSONObject info, RMApp app) throws JSONException, info.getInt("preemptedResourceVCores"), info.getInt("numNonAMContainerPreempted"), info.getInt("numAMContainerPreempted"), - info.getString("logAggregationStatus")); + info.getString("logAggregationStatus"), + info.getBoolean("unmanagedApplication")); } public void verifyAppInfoGeneric(RMApp app, String id, String user, @@ -1342,7 +1344,8 @@ public void verifyAppInfoGeneric(RMApp app, String id, String user, int allocatedMB, int allocatedVCores, int numContainers, int preemptedResourceMB, int preemptedResourceVCores, int numNonAMContainerPreempted, int numAMContainerPreempted, - String logAggregationStatus) throws JSONException, + String logAggregationStatus, boolean unmanagedApplication) + throws JSONException, Exception { WebServicesTestUtils.checkStringMatch("id", app.getApplicationId() @@ -1392,6 +1395,9 @@ public void verifyAppInfoGeneric(RMApp app, String id, String user, assertEquals("Log aggregation Status doesn't match", app .getLogAggregationStatusForAppReport().toString(), logAggregationStatus); + assertEquals("unmanagedApplication doesn't match", app + .getApplicationSubmissionContext().getUnmanagedAM(), + unmanagedApplication); } @Test -- 1.9.2.msysgit.0