diff --git hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestClientRedirect.java hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestClientRedirect.java index 017b993..08d601c 100644 --- hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestClientRedirect.java +++ hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestClientRedirect.java @@ -315,7 +315,8 @@ public GetClusterMetricsResponse getClusterMetrics( @Override public GetAllApplicationsResponse getAllApplications( - GetAllApplicationsRequest request) throws IOException { + GetAllApplicationsRequest request, String applicationType) + throws IOException { return null; } diff --git hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestResourceMgrDelegate.java hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestResourceMgrDelegate.java index 57b09c5..bc5532b 100644 --- hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestResourceMgrDelegate.java +++ hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestResourceMgrDelegate.java @@ -104,9 +104,9 @@ public void tesAllJobs() throws Exception { FinalApplicationStatus.FAILED)); allApplicationsResponse.setApplicationList(applications); Mockito.when( - applicationsManager.getAllApplications(Mockito - .any(GetAllApplicationsRequest.class))).thenReturn( - allApplicationsResponse); + applicationsManager.getAllApplications( + Mockito.any(GetAllApplicationsRequest.class), Mockito.anyString())) + .thenReturn(allApplicationsResponse); ResourceMgrDelegate resourceMgrDelegate = new ResourceMgrDelegate( new YarnConfiguration()) { @Override diff --git hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestYARNRunner.java hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestYARNRunner.java index e25f3cd..76d64e7 100644 --- hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestYARNRunner.java +++ hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestYARNRunner.java @@ -94,6 +94,7 @@ import org.apache.log4j.WriterAppender; import org.junit.Before; import org.junit.Test; +import org.mockito.Mockito; import org.mockito.invocation.InvocationOnMock; import org.mockito.stubbing.Answer; @@ -211,10 +212,14 @@ protected void serviceStart() { verify(clientRMProtocol).forceKillApplication(any(KillApplicationRequest.class)); /* make sure getalljobs calls get all applications */ - when(clientRMProtocol.getAllApplications(any(GetAllApplicationsRequest.class))). - thenReturn(recordFactory.newRecordInstance(GetAllApplicationsResponse.class)); + when( + clientRMProtocol.getAllApplications( + any(GetAllApplicationsRequest.class), Mockito.anyString())) + .thenReturn( + recordFactory.newRecordInstance(GetAllApplicationsResponse.class)); delegate.getAllJobs(); - verify(clientRMProtocol).getAllApplications(any(GetAllApplicationsRequest.class)); + verify(clientRMProtocol).getAllApplications( + any(GetAllApplicationsRequest.class), Mockito.anyString()); /* make sure getapplication report is called */ when(clientRMProtocol.getApplicationReport(any(GetApplicationReportRequest.class))) diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/ClientRMProtocol.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/ClientRMProtocol.java index 593a8ea..84a7d82 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/ClientRMProtocol.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/ClientRMProtocol.java @@ -212,7 +212,7 @@ public GetClusterMetricsResponse getClusterMetrics( * @throws IOException */ public GetAllApplicationsResponse getAllApplications( - GetAllApplicationsRequest request) + GetAllApplicationsRequest request, String applicationType) throws YarnException, IOException; /** diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/GetAllApplicationsRequest.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/GetAllApplicationsRequest.java index b97c8c3..fcb5bae 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/GetAllApplicationsRequest.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/GetAllApplicationsRequest.java @@ -18,9 +18,14 @@ package org.apache.hadoop.yarn.api.protocolrecords; +import java.util.List; + +import org.apache.hadoop.classification.InterfaceAudience.Private; import org.apache.hadoop.classification.InterfaceAudience.Public; import org.apache.hadoop.classification.InterfaceStability.Stable; +import org.apache.hadoop.classification.InterfaceStability.Unstable; import org.apache.hadoop.yarn.api.ClientRMProtocol; +import org.apache.hadoop.yarn.api.records.ApplicationReport; import org.apache.hadoop.yarn.util.Records; /** @@ -29,7 +34,7 @@ * *

Currently, this is empty.

* - * @see ClientRMProtocol#getAllApplications(GetAllApplicationsRequest) + * @see ClientRMProtocol#getAllApplications(GetAllApplicationsRequest, String) */ @Public @Stable @@ -39,4 +44,12 @@ public static GetAllApplicationsRequest newInstance() { Records.newRecord(GetAllApplicationsRequest.class); return request; } + + @Public + @Stable + public abstract String getApplicationType(); + + @Private + @Unstable + public abstract void setApplicationType(String applicationType); } diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/GetAllApplicationsResponse.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/GetAllApplicationsResponse.java index b805e26..0f430d9 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/GetAllApplicationsResponse.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/GetAllApplicationsResponse.java @@ -37,7 +37,7 @@ * is running, RPC port, tracking URL, diagnostics, start time etc.

* * @see ApplicationReport - * @see ClientRMProtocol#getAllApplications(GetAllApplicationsRequest) + * @see ClientRMProtocol#getAllApplications(GetAllApplicationsRequest, String) */ @Public @Stable diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/impl/pb/GetAllApplicationsRequestPBImpl.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/impl/pb/GetAllApplicationsRequestPBImpl.java index e003e5f..39aef76 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/impl/pb/GetAllApplicationsRequestPBImpl.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/impl/pb/GetAllApplicationsRequestPBImpl.java @@ -20,6 +20,7 @@ import org.apache.hadoop.yarn.api.protocolrecords.GetAllApplicationsRequest; import org.apache.hadoop.yarn.proto.YarnServiceProtos.GetAllApplicationsRequestProto; +import org.apache.hadoop.yarn.proto.YarnServiceProtos.GetAllApplicationsRequestProtoOrBuilder; public class GetAllApplicationsRequestPBImpl extends GetAllApplicationsRequest { GetAllApplicationsRequestProto proto = GetAllApplicationsRequestProto.getDefaultInstance(); @@ -60,4 +61,19 @@ public boolean equals(Object other) { public String toString() { return getProto().toString().replaceAll("\\n", ", ").replaceAll("\\s+", " "); } + + @Override + public String getApplicationType() { + GetAllApplicationsRequestProtoOrBuilder p = viaProto ? proto : builder; + return p.getApplicationType(); + } + + @Override + public void setApplicationType(String applicationType) { + if (viaProto || builder == null) { + builder = GetAllApplicationsRequestProto.newBuilder(proto); + } + viaProto = false; + builder.setApplicationType(applicationType); + } } diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java index 3ef06e0..e6135dd 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java @@ -256,6 +256,9 @@ /** Default application type */ public static final String DEFAULT_APPLICATION_TYPE = "YARN"; + /** ALL application type */ + public static final String ALL_APPLICATION_TYPE = "ALL"; + /** Default application type length */ public static final int APPLICATION_TYPE_LENGTH = 20; diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/yarn_service_protos.proto hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/yarn_service_protos.proto index 8e81f21..96729ca 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/yarn_service_protos.proto +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/yarn_service_protos.proto @@ -114,6 +114,7 @@ message GetClusterMetricsResponseProto { } message GetAllApplicationsRequestProto { + optional string applicationType = 1; } message GetAllApplicationsResponseProto { diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/YarnClientImpl.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/YarnClientImpl.java index 48be1a3..350de7f 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/YarnClientImpl.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/YarnClientImpl.java @@ -195,7 +195,9 @@ public ApplicationReport getApplicationReport(ApplicationId appId) throws YarnException, IOException { GetAllApplicationsRequest request = Records.newRecord(GetAllApplicationsRequest.class); - GetAllApplicationsResponse response = rmClient.getAllApplications(request); + GetAllApplicationsResponse response = + rmClient.getAllApplications(request, + YarnConfiguration.ALL_APPLICATION_TYPE); return response.getApplicationList(); } diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/impl/pb/client/ClientRMProtocolPBClientImpl.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/impl/pb/client/ClientRMProtocolPBClientImpl.java index 8394324..ed13261 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/impl/pb/client/ClientRMProtocolPBClientImpl.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/impl/pb/client/ClientRMProtocolPBClientImpl.java @@ -187,8 +187,9 @@ public SubmitApplicationResponse submitApplication( @Override public GetAllApplicationsResponse getAllApplications( - GetAllApplicationsRequest request) throws YarnException, - IOException { + GetAllApplicationsRequest request, String applicationType) + throws YarnException, IOException { + request.setApplicationType(applicationType); GetAllApplicationsRequestProto requestProto = ((GetAllApplicationsRequestPBImpl) request).getProto(); try { diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/impl/pb/service/ClientRMProtocolPBServiceImpl.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/impl/pb/service/ClientRMProtocolPBServiceImpl.java index caad876..9b1cf85 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/impl/pb/service/ClientRMProtocolPBServiceImpl.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/impl/pb/service/ClientRMProtocolPBServiceImpl.java @@ -174,7 +174,8 @@ public GetAllApplicationsResponseProto getAllApplications( GetAllApplicationsRequestPBImpl request = new GetAllApplicationsRequestPBImpl(proto); try { - GetAllApplicationsResponse response = real.getAllApplications(request); + GetAllApplicationsResponse response = + real.getAllApplications(request, request.getApplicationType()); return ((GetAllApplicationsResponsePBImpl)response).getProto(); } catch (YarnException e) { throw new ServiceException(e); diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ClientRMService.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ClientRMService.java index 88bc68b..ab6fd69 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ClientRMService.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ClientRMService.java @@ -393,7 +393,8 @@ public GetClusterMetricsResponse getClusterMetrics( @Override public GetAllApplicationsResponse getAllApplications( - GetAllApplicationsRequest request) throws YarnException { + GetAllApplicationsRequest request, String applicationType) + throws YarnException { UserGroupInformation callerUGI; try { @@ -407,7 +408,12 @@ public GetAllApplicationsResponse getAllApplications( for (RMApp application : this.rmContext.getRMApps().values()) { boolean allowAccess = checkAccess(callerUGI, application.getUser(), ApplicationAccessType.VIEW_APP, application.getApplicationId()); - reports.add(application.createAndGetApplicationReport(allowAccess)); + ApplicationReport applicationReport = + application.createAndGetApplicationReport(allowAccess); + if (applicationType.equals(YarnConfiguration.ALL_APPLICATION_TYPE) + || applicationType.equals(applicationReport.getApplicationType())) { + reports.add(application.createAndGetApplicationReport(allowAccess)); + } } GetAllApplicationsResponse response = diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestApplicationACLs.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestApplicationACLs.java index ef36858..b05b809 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestApplicationACLs.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestApplicationACLs.java @@ -214,7 +214,8 @@ private void verifyOwnerAccess() throws Exception { // List apps as owner Assert.assertEquals("App view by owner should list the apps!!", 1, rmClient.getAllApplications( - recordFactory.newRecordInstance(GetAllApplicationsRequest.class)) + recordFactory.newRecordInstance(GetAllApplicationsRequest.class), + YarnConfiguration.DEFAULT_APPLICATION_TYPE) .getApplicationList().size()); // Kill app as owner @@ -245,7 +246,8 @@ private void verifySuperUserAccess() throws Exception { // List apps as superUser Assert.assertEquals("App view by super-user should list the apps!!", 2, superUserClient.getAllApplications( - recordFactory.newRecordInstance(GetAllApplicationsRequest.class)) + recordFactory.newRecordInstance(GetAllApplicationsRequest.class), + YarnConfiguration.DEFAULT_APPLICATION_TYPE) .getApplicationList().size()); // Kill app as the superUser @@ -276,7 +278,8 @@ private void verifyFriendAccess() throws Exception { // List apps as friend Assert.assertEquals("App view by a friend should list the apps!!", 3, friendClient.getAllApplications( - recordFactory.newRecordInstance(GetAllApplicationsRequest.class)) + recordFactory.newRecordInstance(GetAllApplicationsRequest.class), + YarnConfiguration.DEFAULT_APPLICATION_TYPE) .getApplicationList().size()); // Kill app as the friend @@ -309,7 +312,8 @@ private void verifyEnemyAccess() throws Exception { // List apps as enemy List appReports = enemyRmClient .getAllApplications(recordFactory - .newRecordInstance(GetAllApplicationsRequest.class)) + .newRecordInstance(GetAllApplicationsRequest.class), + YarnConfiguration.DEFAULT_APPLICATION_TYPE) .getApplicationList(); Assert.assertEquals("App view by enemy should list the apps!!", 4, appReports.size());