diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ApplicationReport.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ApplicationReport.java index 26f9978..92a9e22 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ApplicationReport.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ApplicationReport.java @@ -265,4 +265,16 @@ @Private @Unstable void setProgress(float progress); + + /** + * Get the application's Type + * @return application's Type + */ + @Public + @Stable + String getApplicationType(); + + @Private + @Unstable + void setApplicationType(String applicationType); } diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ApplicationSubmissionContext.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ApplicationSubmissionContext.java index 41f1ee7..4941a28 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ApplicationSubmissionContext.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ApplicationSubmissionContext.java @@ -199,4 +199,23 @@ @Public @Stable public void setResource(Resource resource); + + /** + * Get the applicationType is the application type + * + * @return applicationType is the application type + */ + @Public + @Stable + public String getApplicationType(); + + /** + * Set the applicationType is the application type + * + * @param applicationType + * applicationType is the application type + */ + @Public + @Stable + public void setApplicationType(String applicationType); } \ No newline at end of file diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ApplicationReportPBImpl.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ApplicationReportPBImpl.java index de6c41c..115888b 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ApplicationReportPBImpl.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ApplicationReportPBImpl.java @@ -220,6 +220,15 @@ public float getProgress() { } @Override + public String getApplicationType() { + ApplicationReportProtoOrBuilder p = viaProto ? proto : builder; + if (!p.hasApplicationType()) { + return null; + } + return p.getApplicationType(); + } + + @Override public void setApplicationId(ApplicationId applicationId) { maybeInitBuilder(); if (applicationId == null) @@ -318,6 +327,16 @@ public void setUser(String user) { } builder.setUser((user)); } + + @Override + public void setApplicationType(String applicationType) { + maybeInitBuilder(); + if (applicationType == null) { + builder.clearApplicationType(); + return; + } + builder.setApplicationType((applicationType)); + } @Override public void setDiagnostics(String diagnostics) { diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ApplicationSubmissionContextPBImpl.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ApplicationSubmissionContextPBImpl.java index 403ce6e..996b803 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ApplicationSubmissionContextPBImpl.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ApplicationSubmissionContextPBImpl.java @@ -165,6 +165,15 @@ public String getQueue() { } @Override + public String getApplicationType() { + ApplicationSubmissionContextProtoOrBuilder p = viaProto ? proto : builder; + if (!p.hasApplicationType()) { + return null; + } + return (p.getApplicationType()); + } + + @Override public void setQueue(String queue) { maybeInitBuilder(); if (queue == null) { @@ -173,6 +182,16 @@ public void setQueue(String queue) { } builder.setQueue((queue)); } + + @Override + public void setApplicationType(String applicationType) { + maybeInitBuilder(); + if (applicationType == null) { + builder.clearApplicationType(); + return; + } + builder.setApplicationType((applicationType)); + } @Override public ContainerLaunchContext getAMContainerSpec() { diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/yarn_protos.proto hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/yarn_protos.proto index 07cd238..6f8682d 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/yarn_protos.proto +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/yarn_protos.proto @@ -162,6 +162,7 @@ message ApplicationReportProto { optional string originalTrackingUrl = 15; optional ApplicationAttemptIdProto currentApplicationAttemptId = 16; optional float progress = 17; + optional string applicationType = 18; } enum NodeStateProto { @@ -219,6 +220,7 @@ message ApplicationSubmissionContextProto { optional bool unmanaged_am = 7 [default = false]; optional int32 maxAppAttempts = 8 [default = 0]; optional ResourceProto resource = 9; + optional string applicationType = 10 [default = "YARN"]; } enum ApplicationAccessTypeProto { diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/cli/ApplicationCLI.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/cli/ApplicationCLI.java index 1f1703b..582b5ad 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/cli/ApplicationCLI.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/cli/ApplicationCLI.java @@ -35,7 +35,7 @@ public class ApplicationCLI extends YarnCLI { private static final String APPLICATIONS_PATTERN = - "%30s\t%20s\t%10s\t%10s\t%18s\t%18s\t%15s\t%35s" + + "%30s\t%20s\t%20s\t%10s\t%10s\t%18s\t%18s\t%15s\t%35s" + System.getProperty("line.separator"); public static void main(String[] args) throws Exception { @@ -99,16 +99,16 @@ private void listAllApplications() throws YarnRemoteException, IOException { writer.println("Total Applications:" + appsReport.size()); writer.printf(APPLICATIONS_PATTERN, "Application-Id", - "Application-Name", "User", "Queue", "State", "Final-State", - "Progress", "Tracking-URL"); + "Application-Name","Application-Type", "User", "Queue", + "State", "Final-State","Progress", "Tracking-URL"); for (ApplicationReport appReport : appsReport) { DecimalFormat formatter = new DecimalFormat("###.##%"); String progress = formatter.format(appReport.getProgress()); writer.printf(APPLICATIONS_PATTERN, appReport.getApplicationId(), - appReport.getName(), appReport.getUser(), appReport.getQueue(), - appReport.getYarnApplicationState(), appReport - .getFinalApplicationStatus(), - progress, appReport.getOriginalTrackingUrl()); + appReport.getName(),appReport.getApplicationType(), appReport.getUser(), + appReport.getQueue(),appReport.getYarnApplicationState(), + appReport.getFinalApplicationStatus(),progress, + appReport.getOriginalTrackingUrl()); } writer.flush(); } @@ -146,6 +146,8 @@ private void printApplicationReport(String applicationId) appReportStr.println(appReport.getApplicationId()); appReportStr.print("\tApplication-Name : "); appReportStr.println(appReport.getName()); + appReportStr.print("\tApplication-Type : "); + appReportStr.println(appReport.getApplicationType()); appReportStr.print("\tUser : "); appReportStr.println(appReport.getUser()); appReportStr.print("\tQueue : "); diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/TestYarnClient.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/TestYarnClient.java index 098c7d4..c3232d0 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/TestYarnClient.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/TestYarnClient.java @@ -25,6 +25,7 @@ import static org.mockito.Mockito.when; import java.io.IOException; +import java.util.HashMap; import junit.framework.Assert; @@ -32,14 +33,20 @@ import org.apache.hadoop.yarn.api.ClientRMProtocol; import org.apache.hadoop.yarn.api.protocolrecords.GetApplicationReportRequest; import org.apache.hadoop.yarn.api.protocolrecords.GetApplicationReportResponse; +import org.apache.hadoop.yarn.api.records.ApplicationAccessType; import org.apache.hadoop.yarn.api.records.ApplicationId; import org.apache.hadoop.yarn.api.records.ApplicationReport; import org.apache.hadoop.yarn.api.records.ApplicationSubmissionContext; import org.apache.hadoop.yarn.api.records.YarnApplicationState; import org.apache.hadoop.yarn.conf.YarnConfiguration; import org.apache.hadoop.yarn.exceptions.YarnRemoteException; +import org.apache.hadoop.yarn.server.resourcemanager.MockRM; import org.apache.hadoop.yarn.server.resourcemanager.ResourceManager; +import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMApp; import org.apache.hadoop.yarn.util.Records; +import org.apache.log4j.Level; +import org.apache.log4j.LogManager; +import org.apache.log4j.Logger; import org.junit.Test; public class TestYarnClient { @@ -102,6 +109,36 @@ public void testSubmitApplication() { client.stop(); } + @Test(timeout = 30000) + public void testApplicationType() throws Exception { + Logger rootLogger = LogManager.getRootLogger(); + rootLogger.setLevel(Level.DEBUG); + MockRM rm = new MockRM(); + rm.start(); + RMApp app = rm.submitApp(2000); + RMApp app1 = + rm.submitApp(200, "name", "user", + new HashMap(), false, "default", -1, + null, "MAPREDUCE"); + Assert.assertEquals("YARN", app.getApplicationType()); + Assert.assertEquals("MAPREDUCE", app1.getApplicationType()); + rm.stop(); + } + + @Test(timeout = 30000) + public void testApplicationTypeLimit() throws Exception { + Logger rootLogger = LogManager.getRootLogger(); + rootLogger.setLevel(Level.DEBUG); + MockRM rm = new MockRM(); + rm.start(); + RMApp app1 = + rm.submitApp(200, "name", "user", + new HashMap(), false, "default", -1, + null, "MAPREDUCE-LENGTH-IS-20"); + Assert.assertEquals("MAPREDUCE-LENGTH-IS-", app1.getApplicationType()); + rm.stop(); + } + private static class MockYarnClient extends YarnClientImpl { private ApplicationReport mockReport; diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/cli/TestYarnCLI.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/cli/TestYarnCLI.java index 716cf25..4e07ada 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/cli/TestYarnCLI.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/cli/TestYarnCLI.java @@ -76,7 +76,7 @@ public void testGetApplicationReport() throws Exception { applicationId, BuilderUtils.newApplicationAttemptId(applicationId, 1), "user", "queue", "appname", "host", 124, null, YarnApplicationState.FINISHED, "diagnostics", "url", 0, 0, - FinalApplicationStatus.SUCCEEDED, null, "N/A", 0.53789f); + FinalApplicationStatus.SUCCEEDED, null, "N/A", 0.53789f, "YARN"); when(client.getApplicationReport(any(ApplicationId.class))).thenReturn( newApplicationReport); int result = cli.run(new String[] { "-status", applicationId.toString() }); @@ -87,6 +87,7 @@ public void testGetApplicationReport() throws Exception { pw.println("Application Report : "); pw.println("\tApplication-Id : application_1234_0005"); pw.println("\tApplication-Name : appname"); + pw.println("\tApplication-Type : YARN"); pw.println("\tUser : user"); pw.println("\tQueue : queue"); pw.println("\tStart-Time : 0"); @@ -112,7 +113,7 @@ public void testGetAllApplications() throws Exception { applicationId, BuilderUtils.newApplicationAttemptId(applicationId, 1), "user", "queue", "appname", "host", 124, null, YarnApplicationState.FINISHED, "diagnostics", "url", 0, 0, - FinalApplicationStatus.SUCCEEDED, null, "N/A", 0.53789f); + FinalApplicationStatus.SUCCEEDED, null, "N/A", 0.53789f, "YARN"); List applicationReports = new ArrayList(); applicationReports.add(newApplicationReport); when(client.getApplicationList()).thenReturn(applicationReports); @@ -124,11 +125,13 @@ public void testGetAllApplications() throws Exception { PrintWriter pw = new PrintWriter(baos); pw.println("Total Applications:1"); pw.print(" Application-Id\t Application-Name"); + pw.print("\t Application-Type"); pw.print("\t User\t Queue\t State\t "); pw.print("Final-State\t Progress"); pw.println("\t Tracking-URL"); pw.print(" application_1234_0005\t "); - pw.print("appname\t user\t queue\t FINISHED\t "); + pw.print("appname\t YARN\t user\t "); + pw.print("queue\t FINISHED\t "); pw.print("SUCCEEDED\t 53.79%"); pw.println("\t N/A"); pw.close(); diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java index f9b017d..021d403 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java @@ -256,6 +256,12 @@ /** Default application name */ public static final String DEFAULT_APPLICATION_NAME = "N/A"; + /** Default application type */ + public static final String DEFAULT_APPLICATION_TYPE = "YARN"; + + /** Default application type length */ + public static final int APPLICATION_TYPE_LENGTH = 20; + /** Default queue name */ public static final String DEFAULT_QUEUE_NAME = "default"; diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/BuilderUtils.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/BuilderUtils.java index 3e19069..05e0834 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/BuilderUtils.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/BuilderUtils.java @@ -35,6 +35,7 @@ import org.apache.hadoop.yarn.api.records.ApplicationId; import org.apache.hadoop.yarn.api.records.ApplicationReport; import org.apache.hadoop.yarn.api.records.ApplicationResourceUsageReport; +import org.apache.hadoop.yarn.api.records.ApplicationSubmissionContext; import org.apache.hadoop.yarn.api.records.ClientToken; import org.apache.hadoop.yarn.api.records.Container; import org.apache.hadoop.yarn.api.records.ContainerId; @@ -57,6 +58,7 @@ import org.apache.hadoop.yarn.api.records.Token; import org.apache.hadoop.yarn.api.records.URL; import org.apache.hadoop.yarn.api.records.YarnApplicationState; +import org.apache.hadoop.yarn.conf.YarnConfiguration; import org.apache.hadoop.yarn.factories.RecordFactory; import org.apache.hadoop.yarn.factory.providers.RecordFactoryProvider; import org.apache.hadoop.yarn.security.ContainerTokenIdentifier; @@ -336,7 +338,7 @@ public static ApplicationReport newApplicationReport( String url, long startTime, long finishTime, FinalApplicationStatus finalStatus, ApplicationResourceUsageReport appResources, String origTrackingUrl, - float progress) { + float progress, String appType) { ApplicationReport report = recordFactory .newRecordInstance(ApplicationReport.class); report.setApplicationId(applicationId); @@ -356,9 +358,54 @@ public static ApplicationReport newApplicationReport( report.setApplicationResourceUsageReport(appResources); report.setOriginalTrackingUrl(origTrackingUrl); report.setProgress(progress); + report.setApplicationType(appType); return report; } - + + public static ApplicationReport newApplicationReport( + ApplicationId applicationId, ApplicationAttemptId applicationAttemptId, + String user, String queue, String name, String host, int rpcPort, + ClientToken clientToken, YarnApplicationState state, String diagnostics, + String url, long startTime, long finishTime, + FinalApplicationStatus finalStatus, + ApplicationResourceUsageReport appResources, String origTrackingUrl, + float progress) { + return newApplicationReport(applicationId, applicationAttemptId, user, + queue, name, host, rpcPort, clientToken, state, diagnostics, url, + startTime, finishTime, finalStatus, appResources, origTrackingUrl, + progress, YarnConfiguration.DEFAULT_APPLICATION_TYPE); + } + + public static ApplicationSubmissionContext newApplicationSubmissionContext( + ApplicationId applicationId, String applicationName, String queue, + Priority priority, ContainerLaunchContext amContainer, + boolean isUnmanagedAM, boolean cancelTokensWhenComplete, + int maxAppAttempts, Resource resource, String applicationType) { + ApplicationSubmissionContext context = + recordFactory.newRecordInstance(ApplicationSubmissionContext.class); + context.setApplicationId(applicationId); + context.setApplicationName(applicationName); + context.setQueue(queue); + context.setPriority(priority); + context.setAMContainerSpec(amContainer); + context.setUnmanagedAM(isUnmanagedAM); + context.setCancelTokensWhenComplete(cancelTokensWhenComplete); + context.setMaxAppAttempts(maxAppAttempts); + context.setResource(resource); + context.setApplicationType(applicationType); + return context; + } + + public static ApplicationSubmissionContext newApplicationSubmissionContext( + ApplicationId applicationId, String applicationName, String queue, + Priority priority, ContainerLaunchContext amContainer, + boolean isUnmanagedAM, boolean cancelTokensWhenComplete, + int maxAppAttempts, Resource resource) { + return newApplicationSubmissionContext(applicationId, applicationName, + queue, priority, amContainer, isUnmanagedAM, cancelTokensWhenComplete, + maxAppAttempts, resource, null); + } + public static ApplicationResourceUsageReport newApplicationResourceUsageReport( int numUsedContainers, int numReservedContainers, Resource usedResources, Resource reservedResources, Resource neededResources) { diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ClientRMService.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ClientRMService.java index 0e63c11..22df580 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ClientRMService.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ClientRMService.java @@ -298,6 +298,16 @@ public SubmitApplicationResponse submitApplication( submissionContext.setApplicationName( YarnConfiguration.DEFAULT_APPLICATION_NAME); } + if (submissionContext.getApplicationType() == null) { + submissionContext + .setApplicationType(YarnConfiguration.DEFAULT_APPLICATION_TYPE); + } else { + if (submissionContext.getApplicationType().length() > YarnConfiguration.APPLICATION_TYPE_LENGTH) { + submissionContext.setApplicationType(submissionContext + .getApplicationType().substring(0, + YarnConfiguration.APPLICATION_TYPE_LENGTH)); + } + } try { // call RMAppManager to submit application directly diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMAppManager.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMAppManager.java index ebe4562..a8ac80d 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMAppManager.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMAppManager.java @@ -269,7 +269,7 @@ protected void submitApplication( submissionContext.getAMContainerSpec().getUser(), submissionContext.getQueue(), submissionContext, this.scheduler, this.masterService, - submitTime); + submitTime, submissionContext.getApplicationType()); // Concurrent app submissions with same applicationId will fail here // Concurrent app submissions with different applicationIds will not diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/RMApp.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/RMApp.java index 8629ae2..ef11361 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/RMApp.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/RMApp.java @@ -179,4 +179,10 @@ * @return the number of max attempts of the application. */ int getMaxAppAttempts(); + + /** + * Returns the application type + * @return the application type. + */ + String getApplicationType(); } diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/RMAppImpl.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/RMAppImpl.java index 7965fe2..2e14fe8 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/RMAppImpl.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/RMAppImpl.java @@ -97,6 +97,7 @@ = new LinkedHashMap(); private final long submitTime; private final Set updatedNodes = new HashSet(); + private final String applicationType; // Mutable fields private long startTime; @@ -230,7 +231,7 @@ public RMAppImpl(ApplicationId applicationId, RMContext rmContext, Configuration config, String name, String user, String queue, ApplicationSubmissionContext submissionContext, YarnScheduler scheduler, - ApplicationMasterService masterService, long submitTime) { + ApplicationMasterService masterService, long submitTime, String applicationType) { this.applicationId = applicationId; this.name = name; @@ -245,6 +246,7 @@ public RMAppImpl(ApplicationId applicationId, RMContext rmContext, this.masterService = masterService; this.submitTime = submitTime; this.startTime = System.currentTimeMillis(); + this.applicationType = applicationType; int globalMaxAppAttempts = conf.getInt(YarnConfiguration.RM_AM_MAX_ATTEMPTS, YarnConfiguration.DEFAULT_RM_AM_MAX_ATTEMPTS); @@ -472,7 +474,7 @@ public ApplicationReport createAndGetApplicationReport(boolean allowAccess) { this.name, host, rpcPort, clientToken, createApplicationState(this.stateMachine.getCurrentState()), diags, trackingUrl, this.startTime, this.finishTime, finishState, - appUsageReport, origTrackingUrl, progress); + appUsageReport, origTrackingUrl, progress, this.applicationType); } finally { this.readLock.unlock(); } @@ -756,4 +758,9 @@ public RMAppState transition(RMAppImpl app, RMAppEvent event) { } } + + @Override + public String getApplicationType() { + return this.applicationType; + } } diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/AppBlock.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/AppBlock.java index c3593de..15ac1f9 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/AppBlock.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/AppBlock.java @@ -103,6 +103,7 @@ protected void render(Block html) { info("Application Overview"). _("User:", app.getUser()). _("Name:", app.getName()). + _("Type:", app.getApplicationType()). _("State:", app.getState()). _("FinalStatus:", app.getFinalStatus()). _("Started:", Times.format(app.getStartTime())). diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/AppsBlock.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/AppsBlock.java index 6fd35ec..0eb6570 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/AppsBlock.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/AppsBlock.java @@ -56,6 +56,7 @@ th(".id", "ID"). th(".user", "User"). th(".name", "Name"). + th(".type", "Type"). th(".queue", "Queue"). th(".starttime", "StartTime"). th(".finishtime", "FinishTime"). @@ -89,6 +90,8 @@ .append(StringEscapeUtils.escapeJavaScript(StringEscapeUtils.escapeHtml( appInfo.getName()))).append("\",\"") .append(StringEscapeUtils.escapeJavaScript(StringEscapeUtils.escapeHtml( + appInfo.getApplicationType()))).append("\",\"") + .append(StringEscapeUtils.escapeJavaScript(StringEscapeUtils.escapeHtml( appInfo.getQueue()))).append("\",\"") .append(appInfo.getStartTime()).append("\",\"") .append(appInfo.getFinishTime()).append("\",\"") diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/AppInfo.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/AppInfo.java index 0977ec9..fa7b4f7 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/AppInfo.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/AppInfo.java @@ -62,7 +62,8 @@ protected String trackingUrl; protected String diagnostics; protected long clusterId; - + protected String applicationType; + // these are only allowed if acls allow protected long startedTime; protected long finishedTime; @@ -95,6 +96,7 @@ public AppInfo(RMApp app, Boolean hasAccess) { this.trackingUrlPretty = trackingUrlIsNotReady ? "UNASSIGNED" : join( HttpConfig.getSchemePrefix(), trackingUrl); this.applicationId = app.getApplicationId(); + this.applicationType = app.getApplicationType(); this.appIdNum = String.valueOf(app.getApplicationId().getId()); this.id = app.getApplicationId().toString(); this.user = app.getUser().toString(); @@ -215,4 +217,8 @@ public long getClusterId() { return this.clusterId; } + public String getApplicationType() { + return this.applicationType; + } + } diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/MockRM.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/MockRM.java index 48557aa..d40d707 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/MockRM.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/MockRM.java @@ -156,6 +156,13 @@ public RMApp submitApp(int masterMemory, String name, String user, public RMApp submitApp(int masterMemory, String name, String user, Map acls, boolean unmanaged, String queue, int maxAppAttempts, Credentials ts) throws Exception { + return submitApp(masterMemory, name, user, acls, unmanaged, queue, + maxAppAttempts, ts, null); + } + + public RMApp submitApp(int masterMemory, String name, String user, + Map acls, boolean unmanaged, String queue, + int maxAppAttempts, Credentials ts, String appType) throws Exception { ClientRMProtocol client = getClientRMService(); GetNewApplicationResponse resp = client.getNewApplication(Records .newRecord(GetNewApplicationRequest.class)); @@ -174,6 +181,11 @@ public RMApp submitApp(int masterMemory, String name, String user, if (queue != null) { sub.setQueue(queue); } + if (appType == null || appType.equals("")) { + sub.setApplicationType(YarnConfiguration.DEFAULT_APPLICATION_TYPE); + } else { + sub.setApplicationType(appType); + } ContainerLaunchContext clc = Records .newRecord(ContainerLaunchContext.class); final Resource capability = Records.newRecord(Resource.class); diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestClientRMService.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestClientRMService.java index 577ea0a..0141968 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestClientRMService.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestClientRMService.java @@ -449,7 +449,7 @@ private RMAppImpl getRMApp(RMContext rmContext, YarnScheduler yarnScheduler, when(asContext.getMaxAppAttempts()).thenReturn(1); return new RMAppImpl(applicationId3, rmContext, config, null, null, queueName, asContext, yarnScheduler, null , System - .currentTimeMillis()); + .currentTimeMillis(), "YARN"); } private static YarnScheduler mockYarnScheduler() { diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/applicationsmanager/MockAsm.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/applicationsmanager/MockAsm.java index afab180..335283f 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/applicationsmanager/MockAsm.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/applicationsmanager/MockAsm.java @@ -236,6 +236,11 @@ public FinalApplicationStatus getFinalApplicationStatus() { public int pullRMNodeUpdates(Collection updatedNodes) { throw new UnsupportedOperationException("Not supported yet."); } + + @Override + public String getApplicationType() { + throw new UnsupportedOperationException("Not supported yet."); + } } public static RMApp newApplication(int i) { diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/MockRMApp.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/MockRMApp.java index c88cebc..3261dd4 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/MockRMApp.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/MockRMApp.java @@ -29,6 +29,7 @@ import org.apache.hadoop.yarn.api.records.ApplicationId; import org.apache.hadoop.yarn.api.records.ApplicationReport; import org.apache.hadoop.yarn.api.records.impl.pb.ApplicationSubmissionContextPBImpl; +import org.apache.hadoop.yarn.conf.YarnConfiguration; import org.apache.hadoop.yarn.MockApps; import org.apache.hadoop.yarn.server.resourcemanager.rmapp.attempt.RMAppAttempt; import org.apache.hadoop.yarn.server.resourcemanager.rmnode.RMNode; @@ -208,6 +209,11 @@ public FinalApplicationStatus getFinalApplicationStatus() { @Override public int pullRMNodeUpdates(Collection updatedNodes) { throw new UnsupportedOperationException("Not supported yet."); + } + + @Override + public String getApplicationType() { + return YarnConfiguration.DEFAULT_APPLICATION_TYPE; }; } diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/TestRMAppTransitions.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/TestRMAppTransitions.java index 60ea8fc..b117e29 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/TestRMAppTransitions.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/TestRMAppTransitions.java @@ -186,7 +186,7 @@ protected RMApp createNewTestApp(ApplicationSubmissionContext submissionContext) RMApp application = new RMAppImpl(applicationId, rmContext, conf, name, user, queue, submissionContext, scheduler, masterService, - System.currentTimeMillis()); + System.currentTimeMillis(), "YARN"); testAppStartState(applicationId, user, name, queue, application); return application; diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFairScheduler.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFairScheduler.java index 4da11f1..47acf71 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFairScheduler.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFairScheduler.java @@ -1619,7 +1619,7 @@ public void testNotAllowSubmitApplication() throws Exception { RMApp application = new RMAppImpl(applicationId, resourceManager.getRMContext(), conf, name, user, queue, submissionContext, scheduler, masterService, - System.currentTimeMillis()); + System.currentTimeMillis(), "YARN"); resourceManager.getRMContext().getRMApps().putIfAbsent(applicationId, application); application.handle(new RMAppEvent(applicationId, RMAppEventType.START)); diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesApps.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesApps.java index e099151..90d9351 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesApps.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesApps.java @@ -788,6 +788,7 @@ public void verifyAppsXML(NodeList nodes, RMApp app) throws JSONException, WebServicesTestUtils.getXmlString(element, "id"), WebServicesTestUtils.getXmlString(element, "user"), WebServicesTestUtils.getXmlString(element, "name"), + WebServicesTestUtils.getXmlString(element, "applicationType"), WebServicesTestUtils.getXmlString(element, "queue"), WebServicesTestUtils.getXmlString(element, "state"), WebServicesTestUtils.getXmlString(element, "finalStatus"), @@ -807,20 +808,20 @@ public void verifyAppInfo(JSONObject info, RMApp app) throws JSONException, Exception { // 15 because trackingUrl not assigned yet - assertEquals("incorrect number of elements", 15, info.length()); + assertEquals("incorrect number of elements", 16, info.length()); verifyAppInfoGeneric(app, info.getString("id"), info.getString("user"), - info.getString("name"), info.getString("queue"), - info.getString("state"), info.getString("finalStatus"), - (float) info.getDouble("progress"), info.getString("trackingUI"), - info.getString("diagnostics"), info.getLong("clusterId"), - info.getLong("startedTime"), info.getLong("finishedTime"), - info.getLong("elapsedTime"), info.getString("amHostHttpAddress"), - info.getString("amContainerLogs")); + info.getString("name"), info.getString("applicationType"), info.getString("queue"), + info.getString("state"), info.getString("finalStatus"), + (float) info.getDouble("progress"), info.getString("trackingUI"), + info.getString("diagnostics"), info.getLong("clusterId"), + info.getLong("startedTime"), info.getLong("finishedTime"), + info.getLong("elapsedTime"), info.getString("amHostHttpAddress"), + info.getString("amContainerLogs")); } public void verifyAppInfoGeneric(RMApp app, String id, String user, - String name, String queue, String state, String finalStatus, + String name, String type, String queue, String state, String finalStatus, float progress, String trackingUI, String diagnostics, long clusterId, long startedTime, long finishedTime, long elapsedTime, String amHostHttpAddress, String amContainerLogs) throws JSONException, @@ -830,6 +831,8 @@ public void verifyAppInfoGeneric(RMApp app, String id, String user, .toString(), id); WebServicesTestUtils.checkStringMatch("user", app.getUser(), user); WebServicesTestUtils.checkStringMatch("name", app.getName(), name); + WebServicesTestUtils.checkStringMatch("type", app.getApplicationType(), + type); WebServicesTestUtils.checkStringMatch("queue", app.getQueue(), queue); WebServicesTestUtils.checkStringMatch("state", app.getState().toString(), state);