diff --git hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/main/java/org/apache/hadoop/mapred/ResourceMgrDelegate.java hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/main/java/org/apache/hadoop/mapred/ResourceMgrDelegate.java index 349a019..aa27c8b 100644 --- hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/main/java/org/apache/hadoop/mapred/ResourceMgrDelegate.java +++ hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/main/java/org/apache/hadoop/mapred/ResourceMgrDelegate.java @@ -75,7 +75,8 @@ public ResourceMgrDelegate(YarnConfiguration conf) { public JobStatus[] getAllJobs() throws IOException, InterruptedException { try { - return TypeConverter.fromYarnApps(super.getApplicationList(), this.conf); + return TypeConverter.fromYarnApps( + super.getApplicationList(null), this.conf); } catch (YarnException e) { throw new IOException(e); } diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/GetAllApplicationsRequest.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/GetAllApplicationsRequest.java index 61cea28..a8e377f 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/GetAllApplicationsRequest.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/GetAllApplicationsRequest.java @@ -18,8 +18,10 @@ package org.apache.hadoop.yarn.api.protocolrecords; +import org.apache.hadoop.classification.InterfaceAudience.Private; import org.apache.hadoop.classification.InterfaceAudience.Public; import org.apache.hadoop.classification.InterfaceStability.Stable; +import org.apache.hadoop.classification.InterfaceStability.Unstable; import org.apache.hadoop.yarn.api.ApplicationClientProtocol; import org.apache.hadoop.yarn.util.Records; @@ -34,9 +36,28 @@ @Public @Stable public abstract class GetAllApplicationsRequest { - public static GetAllApplicationsRequest newInstance() { + + public static GetAllApplicationsRequest newInstance(String applicationType) { GetAllApplicationsRequest request = Records.newRecord(GetAllApplicationsRequest.class); + request.setApplicationType(applicationType); return request; } + + /** + * Get the application type to filter applications on + * @return Application Type to filter on + */ + @Public + @Stable + public abstract String getApplicationType(); + + /** + * Set the application type to filter applications on + * @param applicationType Application Type to filter on. + * If not defined, match all applications + */ + @Private + @Unstable + public abstract void setApplicationType(String applicationType); } diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/impl/pb/GetAllApplicationsRequestPBImpl.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/impl/pb/GetAllApplicationsRequestPBImpl.java index e003e5f..e623709 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/impl/pb/GetAllApplicationsRequestPBImpl.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/impl/pb/GetAllApplicationsRequestPBImpl.java @@ -20,9 +20,11 @@ import org.apache.hadoop.yarn.api.protocolrecords.GetAllApplicationsRequest; import org.apache.hadoop.yarn.proto.YarnServiceProtos.GetAllApplicationsRequestProto; +import org.apache.hadoop.yarn.proto.YarnServiceProtos.GetAllApplicationsRequestProtoOrBuilder; public class GetAllApplicationsRequestPBImpl extends GetAllApplicationsRequest { - GetAllApplicationsRequestProto proto = GetAllApplicationsRequestProto.getDefaultInstance(); + GetAllApplicationsRequestProto proto = + GetAllApplicationsRequestProto.getDefaultInstance(); GetAllApplicationsRequestProto.Builder builder = null; boolean viaProto = false; @@ -41,6 +43,32 @@ public GetAllApplicationsRequestProto getProto() { return proto; } + private void maybeInitBuilder() { + if (viaProto || builder == null) { + builder = GetAllApplicationsRequestProto.newBuilder(proto); + } + viaProto = false; + } + + @Override + public String getApplicationType() { + GetAllApplicationsRequestProtoOrBuilder p = viaProto ? proto : builder; + if (!p.hasApplicationType()) { + return null; + } + return p.getApplicationType(); + } + + @Override + public void setApplicationType(String applicationType) { + maybeInitBuilder(); + if (applicationType == null) { + builder.clearApplicationType(); + return; + } + builder.setApplicationType(applicationType); + } + @Override public int hashCode() { return getProto().hashCode(); diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/yarn_service_protos.proto hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/yarn_service_protos.proto index 81a7d95..6b2d0d3 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/yarn_service_protos.proto +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/yarn_service_protos.proto @@ -117,6 +117,7 @@ message GetClusterMetricsResponseProto { } message GetAllApplicationsRequestProto { + optional string application_type = 1; } message GetAllApplicationsResponseProto { diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/YarnClient.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/YarnClient.java index 53303b1..9624d13 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/YarnClient.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/YarnClient.java @@ -141,13 +141,16 @@ ApplicationReport getApplicationReport(ApplicationId appId) * then the corresponding report will be filtered as described in * {@link #getApplicationReport(ApplicationId)}. *

- * + * + * @param applicationType Application Type to filter upon. If null, no + * applications are filtered out. + * * @return a list of reports of all running applications * @throws YarnException * @throws IOException */ - List getApplicationList() throws YarnException, - IOException; + List getApplicationList(String applicationType) + throws YarnException, IOException; /** *

diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/YarnClientImpl.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/YarnClientImpl.java index 0717fa4..38fe604 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/YarnClientImpl.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/YarnClientImpl.java @@ -191,11 +191,13 @@ public ApplicationReport getApplicationReport(ApplicationId appId) } @Override - public List getApplicationList() + public List getApplicationList(String applicationType) throws YarnException, IOException { GetAllApplicationsRequest request = Records.newRecord(GetAllApplicationsRequest.class); - GetAllApplicationsResponse response = rmClient.getAllApplications(request); + request.setApplicationType(applicationType); + GetAllApplicationsResponse response = + rmClient.getAllApplications(request); return response.getApplicationList(); } diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/cli/ApplicationCLI.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/cli/ApplicationCLI.java index 6bcd804..4ba5d4f 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/cli/ApplicationCLI.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/cli/ApplicationCLI.java @@ -64,7 +64,11 @@ public int run(String[] args) throws Exception { } printApplicationReport(cliParser.getOptionValue(STATUS_CMD)); } else if (cliParser.hasOption(LIST_CMD)) { - listAllApplications(); + if (args.length == 2) { + listAllApplications(args[1]); + } else { + listAllApplications(null); + } } else if (cliParser.hasOption(KILL_CMD)) { if (args.length != 2) { printUsage(opts); @@ -93,9 +97,11 @@ private void printUsage(Options opts) { * @throws YarnException * @throws IOException */ - private void listAllApplications() throws YarnException, IOException { + private void listAllApplications(String applicationType) + throws YarnException, IOException { PrintWriter writer = new PrintWriter(sysout); - List appsReport = client.getApplicationList(); + List appsReport = + client.getApplicationList(applicationType); writer.println("Total Applications:" + appsReport.size()); writer.printf(APPLICATIONS_PATTERN, "Application-Id", diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/cli/TestYarnCLI.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/cli/TestYarnCLI.java index 49d7867..0c57d7b 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/cli/TestYarnCLI.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/cli/TestYarnCLI.java @@ -115,10 +115,10 @@ public void testGetAllApplications() throws Exception { FinalApplicationStatus.SUCCEEDED, null, "N/A", 0.53789f, "YARN"); List applicationReports = new ArrayList(); applicationReports.add(newApplicationReport); - when(client.getApplicationList()).thenReturn(applicationReports); + when(client.getApplicationList(null)).thenReturn(applicationReports); int result = cli.run(new String[] { "-list" }); assertEquals(0, result); - verify(client).getApplicationList(); + verify(client).getApplicationList(null); ByteArrayOutputStream baos = new ByteArrayOutputStream(); PrintWriter pw = new PrintWriter(baos); diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ClientRMService.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ClientRMService.java index 2a0b482..fcce16c 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ClientRMService.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ClientRMService.java @@ -401,11 +401,18 @@ public GetAllApplicationsResponse getAllApplications( throw RPCUtil.getRemoteException(ie); } + String applicationType = request.getApplicationType(); + boolean bypassFilter = applicationType == null || applicationType.isEmpty(); List reports = new ArrayList(); for (RMApp application : this.rmContext.getRMApps().values()) { boolean allowAccess = checkAccess(callerUGI, application.getUser(), ApplicationAccessType.VIEW_APP, application.getApplicationId()); - reports.add(application.createAndGetApplicationReport(allowAccess)); + ApplicationReport applicationReport = + application.createAndGetApplicationReport(allowAccess); + if (bypassFilter + || applicationType.equals(applicationReport.getApplicationType())) { + reports.add(application.createAndGetApplicationReport(allowAccess)); + } } GetAllApplicationsResponse response = diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestClientRMService.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestClientRMService.java index 1c185ac..afdd818 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestClientRMService.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestClientRMService.java @@ -41,6 +41,8 @@ import org.apache.hadoop.security.token.Token; import org.apache.hadoop.yarn.MockApps; import org.apache.hadoop.yarn.api.ApplicationClientProtocol; +import org.apache.hadoop.yarn.api.protocolrecords.GetAllApplicationsRequest; +import org.apache.hadoop.yarn.api.protocolrecords.GetAllApplicationsResponse; import org.apache.hadoop.yarn.api.protocolrecords.GetApplicationReportRequest; import org.apache.hadoop.yarn.api.protocolrecords.GetApplicationReportResponse; import org.apache.hadoop.yarn.api.protocolrecords.GetClusterNodesRequest; @@ -48,6 +50,7 @@ import org.apache.hadoop.yarn.api.protocolrecords.GetQueueInfoResponse; import org.apache.hadoop.yarn.api.protocolrecords.RenewDelegationTokenRequest; import org.apache.hadoop.yarn.api.protocolrecords.SubmitApplicationRequest; +import org.apache.hadoop.yarn.api.records.ApplicationAccessType; import org.apache.hadoop.yarn.api.records.ApplicationId; import org.apache.hadoop.yarn.api.records.ApplicationReport; import org.apache.hadoop.yarn.api.records.ApplicationSubmissionContext; @@ -87,6 +90,8 @@ private RecordFactory recordFactory = RecordFactoryProvider .getRecordFactory(null); + private String appType = "MockApp"; + private static RMDelegationTokenSecretManager dtsm; @BeforeClass @@ -271,11 +276,18 @@ public void testAppSubmit() throws Exception { new EventHandler() { public void handle(Event event) {} }); + ApplicationId appId1 = getApplicationId(100); + + ApplicationACLsManager mockAclsManager = mock(ApplicationACLsManager.class); + when(mockAclsManager.checkAccess( + UserGroupInformation.getCurrentUser(), + ApplicationAccessType.VIEW_APP, + null, appId1)).thenReturn(true); ClientRMService rmService = - new ClientRMService(rmContext, yarnScheduler, appManager, null, null); + new ClientRMService(rmContext, yarnScheduler, appManager, + mockAclsManager, null); // without name and queue - ApplicationId appId1 = getApplicationId(100); SubmitApplicationRequest submitRequest1 = mockSubmitAppRequest( appId1, null, null); try { @@ -296,6 +308,8 @@ public void handle(Event event) {} ApplicationId appId2 = getApplicationId(101); SubmitApplicationRequest submitRequest2 = mockSubmitAppRequest( appId2, name, queue); + submitRequest2.getApplicationSubmissionContext() + .setApplicationType("matchType"); try { rmService.submitApplication(submitRequest2); } catch (YarnException e) { @@ -314,6 +328,22 @@ public void handle(Event event) {} Assert.assertTrue("The thrown exception is not expected.", e.getMessage().contains("Cannot add a duplicate!")); } + + GetAllApplicationsRequest getAllAppsRequest = + GetAllApplicationsRequest.newInstance(null); + GetAllApplicationsResponse getAllApplicationsResponse = + rmService.getAllApplications(getAllAppsRequest); + Assert.assertEquals(5, + getAllApplicationsResponse.getApplicationList().size()); + + getAllAppsRequest = GetAllApplicationsRequest.newInstance("matchType"); + getAllApplicationsResponse = + rmService.getAllApplications(getAllAppsRequest); + Assert.assertEquals(1, + getAllApplicationsResponse.getApplicationList().size()); + Assert.assertEquals(appId2, + getAllApplicationsResponse.getApplicationList() + .get(0).getApplicationId()); } @Test(timeout=4000) @@ -395,6 +425,7 @@ private SubmitApplicationRequest mockSubmitAppRequest(ApplicationId appId, submissionContext.setQueue(queue); submissionContext.setApplicationId(appId); submissionContext.setResource(resource); + submissionContext.setApplicationType(appType); SubmitApplicationRequest submitRequest = recordFactory.newRecordInstance(SubmitApplicationRequest.class);