diff --git hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/main/java/org/apache/hadoop/mapred/ResourceMgrDelegate.java hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/main/java/org/apache/hadoop/mapred/ResourceMgrDelegate.java index 349a019..aa27c8b 100644 --- hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/main/java/org/apache/hadoop/mapred/ResourceMgrDelegate.java +++ hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/main/java/org/apache/hadoop/mapred/ResourceMgrDelegate.java @@ -75,7 +75,8 @@ public ResourceMgrDelegate(YarnConfiguration conf) { public JobStatus[] getAllJobs() throws IOException, InterruptedException { try { - return TypeConverter.fromYarnApps(super.getApplicationList(), this.conf); + return TypeConverter.fromYarnApps( + super.getApplicationList(null), this.conf); } catch (YarnException e) { throw new IOException(e); } diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/GetAllApplicationsRequest.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/GetAllApplicationsRequest.java index 61cea28..a8e377f 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/GetAllApplicationsRequest.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/GetAllApplicationsRequest.java @@ -18,8 +18,10 @@ package org.apache.hadoop.yarn.api.protocolrecords; +import org.apache.hadoop.classification.InterfaceAudience.Private; import org.apache.hadoop.classification.InterfaceAudience.Public; import org.apache.hadoop.classification.InterfaceStability.Stable; +import org.apache.hadoop.classification.InterfaceStability.Unstable; import org.apache.hadoop.yarn.api.ApplicationClientProtocol; import org.apache.hadoop.yarn.util.Records; @@ -34,9 +36,28 @@ @Public @Stable public abstract class GetAllApplicationsRequest { - public static GetAllApplicationsRequest newInstance() { + + public static GetAllApplicationsRequest newInstance(String applicationType) { GetAllApplicationsRequest request = Records.newRecord(GetAllApplicationsRequest.class); + request.setApplicationType(applicationType); return request; } + + /** + * Get the application type to filter applications on + * @return Application Type to filter on + */ + @Public + @Stable + public abstract String getApplicationType(); + + /** + * Set the application type to filter applications on + * @param applicationType Application Type to filter on. + * If not defined, match all applications + */ + @Private + @Unstable + public abstract void setApplicationType(String applicationType); } diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/impl/pb/GetAllApplicationsRequestPBImpl.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/impl/pb/GetAllApplicationsRequestPBImpl.java index e003e5f..e623709 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/impl/pb/GetAllApplicationsRequestPBImpl.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/impl/pb/GetAllApplicationsRequestPBImpl.java @@ -20,9 +20,11 @@ import org.apache.hadoop.yarn.api.protocolrecords.GetAllApplicationsRequest; import org.apache.hadoop.yarn.proto.YarnServiceProtos.GetAllApplicationsRequestProto; +import org.apache.hadoop.yarn.proto.YarnServiceProtos.GetAllApplicationsRequestProtoOrBuilder; public class GetAllApplicationsRequestPBImpl extends GetAllApplicationsRequest { - GetAllApplicationsRequestProto proto = GetAllApplicationsRequestProto.getDefaultInstance(); + GetAllApplicationsRequestProto proto = + GetAllApplicationsRequestProto.getDefaultInstance(); GetAllApplicationsRequestProto.Builder builder = null; boolean viaProto = false; @@ -41,6 +43,32 @@ public GetAllApplicationsRequestProto getProto() { return proto; } + private void maybeInitBuilder() { + if (viaProto || builder == null) { + builder = GetAllApplicationsRequestProto.newBuilder(proto); + } + viaProto = false; + } + + @Override + public String getApplicationType() { + GetAllApplicationsRequestProtoOrBuilder p = viaProto ? proto : builder; + if (!p.hasApplicationType()) { + return null; + } + return p.getApplicationType(); + } + + @Override + public void setApplicationType(String applicationType) { + maybeInitBuilder(); + if (applicationType == null) { + builder.clearApplicationType(); + return; + } + builder.setApplicationType(applicationType); + } + @Override public int hashCode() { return getProto().hashCode(); diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/yarn_service_protos.proto hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/yarn_service_protos.proto index dc2b100..4cf6a52 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/yarn_service_protos.proto +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/yarn_service_protos.proto @@ -118,6 +118,7 @@ message GetClusterMetricsResponseProto { } message GetAllApplicationsRequestProto { + optional string application_type = 1; } message GetAllApplicationsResponseProto { diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/YarnClient.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/YarnClient.java index 57ef805..9c52b63 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/YarnClient.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/YarnClient.java @@ -178,12 +178,15 @@ public abstract ApplicationReport getApplicationReport(ApplicationId appId) * {@link #getApplicationReport(ApplicationId)}. *

* + * @param applicationType Application Type to filter upon. If null, no + * applications are filtered out. + * * @return a list of reports of all running applications * @throws YarnException * @throws IOException */ - public abstract List getApplicationList() throws YarnException, - IOException; + public abstract List getApplicationList( + String applicationType) throws YarnException, IOException; /** *

diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/YarnClientImpl.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/YarnClientImpl.java index fc41bac..88d37d4 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/YarnClientImpl.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/YarnClientImpl.java @@ -190,11 +190,12 @@ public ApplicationReport getApplicationReport(ApplicationId appId) } @Override - public List getApplicationList() + public List getApplicationList(String applicationType) throws YarnException, IOException { GetAllApplicationsRequest request = - Records.newRecord(GetAllApplicationsRequest.class); - GetAllApplicationsResponse response = rmClient.getAllApplications(request); + GetAllApplicationsRequest.newInstance(applicationType); + GetAllApplicationsResponse response = + rmClient.getAllApplications(request); return response.getApplicationList(); } diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/cli/ApplicationCLI.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/cli/ApplicationCLI.java index 6bcd804..f05903a 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/cli/ApplicationCLI.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/cli/ApplicationCLI.java @@ -26,6 +26,7 @@ import org.apache.commons.cli.CommandLine; import org.apache.commons.cli.GnuParser; import org.apache.commons.cli.HelpFormatter; +import org.apache.commons.cli.Option; import org.apache.commons.cli.Options; import org.apache.hadoop.util.ToolRunner; import org.apache.hadoop.yarn.api.records.ApplicationId; @@ -52,8 +53,12 @@ public int run(String[] args) throws Exception { Options opts = new Options(); opts.addOption(STATUS_CMD, true, "Prints the status of the application."); - opts.addOption(LIST_CMD, false, "Lists all the Applications from RM."); opts.addOption(KILL_CMD, true, "Kills the application."); + opts.addOption(HELP_CMD, false, "Displays help for all commands"); + Option list_opt = new Option(LIST_CMD, true, "Lists all the applications or " + + "ones matching optionally provided application type from RM"); + list_opt.setOptionalArg(true); + opts.addOption(list_opt); CommandLine cliParser = new GnuParser().parse(opts, args); int exitCode = -1; @@ -64,13 +69,20 @@ public int run(String[] args) throws Exception { } printApplicationReport(cliParser.getOptionValue(STATUS_CMD)); } else if (cliParser.hasOption(LIST_CMD)) { - listAllApplications(); + if (args.length > 2) { + printUsage(opts); + return exitCode; + } + listAllApplications(cliParser.getOptionValue(LIST_CMD)); } else if (cliParser.hasOption(KILL_CMD)) { if (args.length != 2) { printUsage(opts); return exitCode; } killApplication(cliParser.getOptionValue(KILL_CMD)); + } else if (cliParser.hasOption(HELP_CMD)) { + printUsage(opts); + return 0; } else { syserr.println("Invalid Command Usage : "); printUsage(opts); @@ -93,9 +105,11 @@ private void printUsage(Options opts) { * @throws YarnException * @throws IOException */ - private void listAllApplications() throws YarnException, IOException { + private void listAllApplications(String applicationType) + throws YarnException, IOException { PrintWriter writer = new PrintWriter(sysout); - List appsReport = client.getApplicationList(); + List appsReport = + client.getApplicationList(applicationType); writer.println("Total Applications:" + appsReport.size()); writer.printf(APPLICATIONS_PATTERN, "Application-Id", diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/cli/YarnCLI.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/cli/YarnCLI.java index aa7cb8d..4cb6799 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/cli/YarnCLI.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/cli/YarnCLI.java @@ -30,6 +30,7 @@ public static final String STATUS_CMD = "status"; public static final String LIST_CMD = "list"; public static final String KILL_CMD = "kill"; + public static final String HELP_CMD = "help"; protected PrintStream sysout; protected PrintStream syserr; protected YarnClient client; diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/cli/TestYarnCLI.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/cli/TestYarnCLI.java index 49d7867..5afdad5 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/cli/TestYarnCLI.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/cli/TestYarnCLI.java @@ -115,10 +115,23 @@ public void testGetAllApplications() throws Exception { FinalApplicationStatus.SUCCEEDED, null, "N/A", 0.53789f, "YARN"); List applicationReports = new ArrayList(); applicationReports.add(newApplicationReport); - when(client.getApplicationList()).thenReturn(applicationReports); - int result = cli.run(new String[] { "-list" }); + + ApplicationId applicationId2 = ApplicationId.newInstance(1234, 6); + ApplicationReport newApplicationReport2 = ApplicationReport.newInstance( + applicationId2, ApplicationAttemptId.newInstance(applicationId2, 2), + "user2", "queue2", "appname2", "host2", 125, null, + YarnApplicationState.FINISHED, "diagnostics2", "url2", 2, 2, + FinalApplicationStatus.SUCCEEDED, null, "N/A", 0.63789f, "NON-YARN"); + applicationReports.add(newApplicationReport2); + + when(client.getApplicationList("YARN")).thenReturn( + getApplicationReports(applicationReports, "YARN")); + when(client.getApplicationList(null)).thenReturn( + getApplicationReports(applicationReports, null)); + + int result = cli.run(new String[] { "-list", "YARN" }); assertEquals(0, result); - verify(client).getApplicationList(); + verify(client).getApplicationList("YARN"); ByteArrayOutputStream baos = new ByteArrayOutputStream(); PrintWriter pw = new PrintWriter(baos); @@ -137,6 +150,49 @@ public void testGetAllApplications() throws Exception { String appsReportStr = baos.toString("UTF-8"); Assert.assertEquals(appsReportStr, sysOutStream.toString()); verify(sysOut, times(1)).write(any(byte[].class), anyInt(), anyInt()); + + sysOutStream.reset(); + result = cli.run(new String[] { "-list" }); + assertEquals(0, result); + verify(client).getApplicationList(null); + + baos = new ByteArrayOutputStream(); + pw = new PrintWriter(baos); + pw.println("Total Applications:2"); + pw.print(" Application-Id\t Application-Name"); + pw.print("\t Application-Type"); + pw.print("\t User\t Queue\t State\t "); + pw.print("Final-State\t Progress"); + pw.println("\t Tracking-URL"); + pw.print(" application_1234_0005\t "); + pw.print("appname\t YARN\t user\t "); + pw.print("queue\t FINISHED\t "); + pw.print("SUCCEEDED\t 53.79%"); + pw.println("\t N/A"); + pw.print(" application_1234_0006\t "); + pw.print("appname2\t NON-YARN\t user2\t "); + pw.print("queue2\t FINISHED\t "); + pw.print("SUCCEEDED\t 63.79%"); + pw.println("\t N/A"); + pw.close(); + appsReportStr = baos.toString("UTF-8"); + Assert.assertEquals(appsReportStr, sysOutStream.toString()); + verify(sysOut, times(2)).write(any(byte[].class), anyInt(), anyInt()); + + } + + private List getApplicationReports( + List applicationReports, String applicationType) { + List appReports = new ArrayList(); + boolean bypassFilter = applicationType == null || applicationType.isEmpty(); + for (ApplicationReport appReport : applicationReports) { + if (!(bypassFilter || appReport.getApplicationType().equals( + applicationType))) { + continue; + } + appReports.add(appReport); + } + return appReports; } @Test diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ClientRMService.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ClientRMService.java index 512ba83..0fc239a 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ClientRMService.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ClientRMService.java @@ -401,8 +401,14 @@ public GetAllApplicationsResponse getAllApplications( throw RPCUtil.getRemoteException(ie); } + String applicationType = request.getApplicationType(); + boolean bypassFilter = applicationType == null || applicationType.isEmpty(); List reports = new ArrayList(); for (RMApp application : this.rmContext.getRMApps().values()) { + String appType = application.getApplicationType(); + if (!(bypassFilter || applicationType.equals(appType))) { + continue; + } boolean allowAccess = checkAccess(callerUGI, application.getUser(), ApplicationAccessType.VIEW_APP, application.getApplicationId()); reports.add(application.createAndGetApplicationReport(allowAccess)); diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestClientRMService.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestClientRMService.java index 1c185ac..afdd818 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestClientRMService.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestClientRMService.java @@ -41,6 +41,8 @@ import org.apache.hadoop.security.token.Token; import org.apache.hadoop.yarn.MockApps; import org.apache.hadoop.yarn.api.ApplicationClientProtocol; +import org.apache.hadoop.yarn.api.protocolrecords.GetAllApplicationsRequest; +import org.apache.hadoop.yarn.api.protocolrecords.GetAllApplicationsResponse; import org.apache.hadoop.yarn.api.protocolrecords.GetApplicationReportRequest; import org.apache.hadoop.yarn.api.protocolrecords.GetApplicationReportResponse; import org.apache.hadoop.yarn.api.protocolrecords.GetClusterNodesRequest; @@ -48,6 +50,7 @@ import org.apache.hadoop.yarn.api.protocolrecords.GetQueueInfoResponse; import org.apache.hadoop.yarn.api.protocolrecords.RenewDelegationTokenRequest; import org.apache.hadoop.yarn.api.protocolrecords.SubmitApplicationRequest; +import org.apache.hadoop.yarn.api.records.ApplicationAccessType; import org.apache.hadoop.yarn.api.records.ApplicationId; import org.apache.hadoop.yarn.api.records.ApplicationReport; import org.apache.hadoop.yarn.api.records.ApplicationSubmissionContext; @@ -87,6 +90,8 @@ private RecordFactory recordFactory = RecordFactoryProvider .getRecordFactory(null); + private String appType = "MockApp"; + private static RMDelegationTokenSecretManager dtsm; @BeforeClass @@ -271,11 +276,18 @@ public void testAppSubmit() throws Exception { new EventHandler() { public void handle(Event event) {} }); + ApplicationId appId1 = getApplicationId(100); + + ApplicationACLsManager mockAclsManager = mock(ApplicationACLsManager.class); + when(mockAclsManager.checkAccess( + UserGroupInformation.getCurrentUser(), + ApplicationAccessType.VIEW_APP, + null, appId1)).thenReturn(true); ClientRMService rmService = - new ClientRMService(rmContext, yarnScheduler, appManager, null, null); + new ClientRMService(rmContext, yarnScheduler, appManager, + mockAclsManager, null); // without name and queue - ApplicationId appId1 = getApplicationId(100); SubmitApplicationRequest submitRequest1 = mockSubmitAppRequest( appId1, null, null); try { @@ -296,6 +308,8 @@ public void handle(Event event) {} ApplicationId appId2 = getApplicationId(101); SubmitApplicationRequest submitRequest2 = mockSubmitAppRequest( appId2, name, queue); + submitRequest2.getApplicationSubmissionContext() + .setApplicationType("matchType"); try { rmService.submitApplication(submitRequest2); } catch (YarnException e) { @@ -314,6 +328,22 @@ public void handle(Event event) {} Assert.assertTrue("The thrown exception is not expected.", e.getMessage().contains("Cannot add a duplicate!")); } + + GetAllApplicationsRequest getAllAppsRequest = + GetAllApplicationsRequest.newInstance(null); + GetAllApplicationsResponse getAllApplicationsResponse = + rmService.getAllApplications(getAllAppsRequest); + Assert.assertEquals(5, + getAllApplicationsResponse.getApplicationList().size()); + + getAllAppsRequest = GetAllApplicationsRequest.newInstance("matchType"); + getAllApplicationsResponse = + rmService.getAllApplications(getAllAppsRequest); + Assert.assertEquals(1, + getAllApplicationsResponse.getApplicationList().size()); + Assert.assertEquals(appId2, + getAllApplicationsResponse.getApplicationList() + .get(0).getApplicationId()); } @Test(timeout=4000) @@ -395,6 +425,7 @@ private SubmitApplicationRequest mockSubmitAppRequest(ApplicationId appId, submissionContext.setQueue(queue); submissionContext.setApplicationId(appId); submissionContext.setResource(resource); + submissionContext.setApplicationType(appType); SubmitApplicationRequest submitRequest = recordFactory.newRecordInstance(SubmitApplicationRequest.class);