diff --git hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/main/java/org/apache/hadoop/mapred/ResourceMgrDelegate.java hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/main/java/org/apache/hadoop/mapred/ResourceMgrDelegate.java index c687de6..9a3d568 100644 --- hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/main/java/org/apache/hadoop/mapred/ResourceMgrDelegate.java +++ hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/main/java/org/apache/hadoop/mapred/ResourceMgrDelegate.java @@ -20,7 +20,9 @@ import java.io.IOException; import java.net.InetSocketAddress; +import java.util.HashSet; import java.util.List; +import java.util.Set; import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; @@ -131,7 +133,10 @@ protected void serviceStop() throws Exception { public JobStatus[] getAllJobs() throws IOException, InterruptedException { try { - return TypeConverter.fromYarnApps(client.getApplicationList(), this.conf); + Set appTypes = new HashSet(1); + appTypes.add(MRJobConfig.MR_APPLICATION_TYPE); + return TypeConverter.fromYarnApps( + client.getApplicationList(appTypes), this.conf); } catch (YarnException e) { throw new IOException(e); } @@ -305,6 +310,12 @@ public ApplicationReport getApplicationReport(ApplicationId appId) } @Override + public List getApplicationList( + Set applicationTypes) throws YarnException, IOException { + return client.getApplicationList(applicationTypes); + } + + @Override public YarnClusterMetrics getYarnClusterMetrics() throws YarnException, IOException { return client.getYarnClusterMetrics(); diff --git hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestClientRedirect.java hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestClientRedirect.java index a60cc4a..1f0b89f 100644 --- hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestClientRedirect.java +++ hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestClientRedirect.java @@ -72,8 +72,8 @@ import org.apache.hadoop.yarn.api.ApplicationClientProtocol; import org.apache.hadoop.yarn.api.protocolrecords.CancelDelegationTokenRequest; import org.apache.hadoop.yarn.api.protocolrecords.CancelDelegationTokenResponse; -import org.apache.hadoop.yarn.api.protocolrecords.GetAllApplicationsRequest; -import org.apache.hadoop.yarn.api.protocolrecords.GetAllApplicationsResponse; +import org.apache.hadoop.yarn.api.protocolrecords.GetApplicationsRequest; +import org.apache.hadoop.yarn.api.protocolrecords.GetApplicationsResponse; import org.apache.hadoop.yarn.api.protocolrecords.GetApplicationReportRequest; import org.apache.hadoop.yarn.api.protocolrecords.GetApplicationReportResponse; import org.apache.hadoop.yarn.api.protocolrecords.GetClusterMetricsRequest; @@ -314,8 +314,8 @@ public GetClusterMetricsResponse getClusterMetrics( } @Override - public GetAllApplicationsResponse getAllApplications( - GetAllApplicationsRequest request) throws IOException { + public GetApplicationsResponse getApplications( + GetApplicationsRequest request) throws IOException { return null; } diff --git hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestResourceMgrDelegate.java hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestResourceMgrDelegate.java index 2339fb5..6c6df66 100644 --- hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestResourceMgrDelegate.java +++ hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestResourceMgrDelegate.java @@ -27,8 +27,8 @@ import org.apache.hadoop.mapreduce.JobStatus; import org.apache.hadoop.mapreduce.JobStatus.State; import org.apache.hadoop.yarn.api.ApplicationClientProtocol; -import org.apache.hadoop.yarn.api.protocolrecords.GetAllApplicationsRequest; -import org.apache.hadoop.yarn.api.protocolrecords.GetAllApplicationsResponse; +import org.apache.hadoop.yarn.api.protocolrecords.GetApplicationsRequest; +import org.apache.hadoop.yarn.api.protocolrecords.GetApplicationsResponse; import org.apache.hadoop.yarn.api.protocolrecords.GetQueueInfoRequest; import org.apache.hadoop.yarn.api.protocolrecords.GetQueueInfoResponse; import org.apache.hadoop.yarn.api.records.ApplicationId; @@ -93,8 +93,8 @@ protected void serviceStart() throws Exception { @Test public void tesAllJobs() throws Exception { final ApplicationClientProtocol applicationsManager = Mockito.mock(ApplicationClientProtocol.class); - GetAllApplicationsResponse allApplicationsResponse = Records - .newRecord(GetAllApplicationsResponse.class); + GetApplicationsResponse allApplicationsResponse = Records + .newRecord(GetApplicationsResponse.class); List applications = new ArrayList(); applications.add(getApplicationReport(YarnApplicationState.FINISHED, FinalApplicationStatus.FAILED)); @@ -106,8 +106,8 @@ public void tesAllJobs() throws Exception { FinalApplicationStatus.FAILED)); allApplicationsResponse.setApplicationList(applications); Mockito.when( - applicationsManager.getAllApplications(Mockito - .any(GetAllApplicationsRequest.class))).thenReturn( + applicationsManager.getApplications(Mockito + .any(GetApplicationsRequest.class))).thenReturn( allApplicationsResponse); ResourceMgrDelegate resourceMgrDelegate = new ResourceMgrDelegate( new YarnConfiguration()) { diff --git hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestYARNRunner.java hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestYARNRunner.java index 0046c9b..0c08b81 100644 --- hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestYARNRunner.java +++ hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestYARNRunner.java @@ -60,8 +60,8 @@ import org.apache.hadoop.security.UserGroupInformation; import org.apache.hadoop.security.token.Token; import org.apache.hadoop.yarn.api.ApplicationClientProtocol; -import org.apache.hadoop.yarn.api.protocolrecords.GetAllApplicationsRequest; -import org.apache.hadoop.yarn.api.protocolrecords.GetAllApplicationsResponse; +import org.apache.hadoop.yarn.api.protocolrecords.GetApplicationsRequest; +import org.apache.hadoop.yarn.api.protocolrecords.GetApplicationsResponse; import org.apache.hadoop.yarn.api.protocolrecords.GetApplicationReportRequest; import org.apache.hadoop.yarn.api.protocolrecords.GetApplicationReportResponse; import org.apache.hadoop.yarn.api.protocolrecords.GetClusterMetricsRequest; @@ -213,10 +213,10 @@ protected void serviceStart() throws Exception { verify(clientRMProtocol).forceKillApplication(any(KillApplicationRequest.class)); /* make sure getalljobs calls get all applications */ - when(clientRMProtocol.getAllApplications(any(GetAllApplicationsRequest.class))). - thenReturn(recordFactory.newRecordInstance(GetAllApplicationsResponse.class)); + when(clientRMProtocol.getApplications(any(GetApplicationsRequest.class))). + thenReturn(recordFactory.newRecordInstance(GetApplicationsResponse.class)); delegate.getAllJobs(); - verify(clientRMProtocol).getAllApplications(any(GetAllApplicationsRequest.class)); + verify(clientRMProtocol).getApplications(any(GetApplicationsRequest.class)); /* make sure getapplication report is called */ when(clientRMProtocol.getApplicationReport(any(GetApplicationReportRequest.class))) diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/ApplicationClientProtocol.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/ApplicationClientProtocol.java index 5427d3b..d6117d2 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/ApplicationClientProtocol.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/ApplicationClientProtocol.java @@ -26,8 +26,8 @@ import org.apache.hadoop.classification.InterfaceStability.Unstable; import org.apache.hadoop.yarn.api.protocolrecords.CancelDelegationTokenRequest; import org.apache.hadoop.yarn.api.protocolrecords.CancelDelegationTokenResponse; -import org.apache.hadoop.yarn.api.protocolrecords.GetAllApplicationsRequest; -import org.apache.hadoop.yarn.api.protocolrecords.GetAllApplicationsResponse; +import org.apache.hadoop.yarn.api.protocolrecords.GetApplicationsRequest; +import org.apache.hadoop.yarn.api.protocolrecords.GetApplicationsResponse; import org.apache.hadoop.yarn.api.protocolrecords.GetApplicationReportRequest; import org.apache.hadoop.yarn.api.protocolrecords.GetApplicationReportResponse; import org.apache.hadoop.yarn.api.protocolrecords.GetClusterMetricsRequest; @@ -205,27 +205,29 @@ public GetClusterMetricsResponse getClusterMetrics( throws YarnException, IOException; /** - *

The interface used by clients to get a report of all Applications + *

The interface used by clients to get a report of Applications + * matching the given application types defined in the request object * in the cluster from the ResourceManager.

* *

The ResourceManager responds with a - * {@link GetAllApplicationsResponse} which includes the - * {@link ApplicationReport} for all the applications.

+ * {@link GetApplicationsResponse} which includes the + * {@link ApplicationReport} for the applications.

* *

If the user does not have VIEW_APP access for an * application then the corresponding report will be filtered as * described in {@link #getApplicationReport(GetApplicationReportRequest)}. *

* - * @param request request for report on all running applications - * @return report on all running applications + * @param request request for report on running applications + * @return report on applications matching the given application types + * defined in the request * @throws YarnException * @throws IOException */ @Public @Stable - public GetAllApplicationsResponse getAllApplications( - GetAllApplicationsRequest request) + public GetApplicationsResponse getApplications( + GetApplicationsRequest request) throws YarnException, IOException; /** diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/GetAllApplicationsRequest.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/GetAllApplicationsRequest.java deleted file mode 100644 index 8890238..0000000 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/GetAllApplicationsRequest.java +++ /dev/null @@ -1,44 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.hadoop.yarn.api.protocolrecords; - -import org.apache.hadoop.classification.InterfaceAudience.Public; -import org.apache.hadoop.classification.InterfaceStability.Stable; -import org.apache.hadoop.yarn.api.ApplicationClientProtocol; -import org.apache.hadoop.yarn.util.Records; - -/** - *

The request from clients to get a report of all Applications - * in the cluster from the ResourceManager.

- * - *

Currently, this is empty.

- * - * @see ApplicationClientProtocol#getAllApplications(GetAllApplicationsRequest) - */ -@Public -@Stable -public abstract class GetAllApplicationsRequest { - @Public - @Stable - public static GetAllApplicationsRequest newInstance() { - GetAllApplicationsRequest request = - Records.newRecord(GetAllApplicationsRequest.class); - return request; - } -} diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/GetAllApplicationsResponse.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/GetAllApplicationsResponse.java deleted file mode 100644 index ae04624..0000000 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/GetAllApplicationsResponse.java +++ /dev/null @@ -1,66 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.hadoop.yarn.api.protocolrecords; - -import java.util.List; - -import org.apache.hadoop.classification.InterfaceAudience.Private; -import org.apache.hadoop.classification.InterfaceAudience.Public; -import org.apache.hadoop.classification.InterfaceStability.Stable; -import org.apache.hadoop.classification.InterfaceStability.Unstable; -import org.apache.hadoop.yarn.api.ApplicationClientProtocol; -import org.apache.hadoop.yarn.api.records.ApplicationReport; -import org.apache.hadoop.yarn.util.Records; - -/** - *

The response sent by the ResourceManager to a client - * requesting an {@link ApplicationReport} for all applications.

- * - *

The ApplicationReport for each application includes details - * such as user, queue, name, host on which the ApplicationMaster - * is running, RPC port, tracking URL, diagnostics, start time etc.

- * - * @see ApplicationReport - * @see ApplicationClientProtocol#getAllApplications(GetAllApplicationsRequest) - */ -@Public -@Stable -public abstract class GetAllApplicationsResponse { - @Private - @Unstable - public static GetAllApplicationsResponse newInstance( - List applications) { - GetAllApplicationsResponse response = - Records.newRecord(GetAllApplicationsResponse.class); - response.setApplicationList(applications); - return response; - } - - /** - * Get ApplicationReport for all applications. - * @return ApplicationReport for all applications - */ - @Public - @Stable - public abstract List getApplicationList(); - - @Private - @Unstable - public abstract void setApplicationList(List applications); -} diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/GetApplicationsRequest.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/GetApplicationsRequest.java new file mode 100644 index 0000000..9a73221 --- /dev/null +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/GetApplicationsRequest.java @@ -0,0 +1,78 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.yarn.api.protocolrecords; + +import java.util.Set; + +import org.apache.hadoop.classification.InterfaceAudience.Private; +import org.apache.hadoop.classification.InterfaceAudience.Public; +import org.apache.hadoop.classification.InterfaceStability.Stable; +import org.apache.hadoop.classification.InterfaceStability.Unstable; +import org.apache.hadoop.yarn.api.ApplicationClientProtocol; +import org.apache.hadoop.yarn.util.Records; + +/** + *

The request from clients to get a report of Applications + * in the cluster from the ResourceManager.

+ * + * + * @see ApplicationClientProtocol#getApplications(GetApplicationsRequest) + */ +@Public +@Stable +public abstract class GetApplicationsRequest { + @Public + @Stable + public static GetApplicationsRequest newInstance() { + GetApplicationsRequest request = + Records.newRecord(GetApplicationsRequest.class); + return request; + } + + @Public + @Stable + public static GetApplicationsRequest newInstance( + Set applicationTypes) { + GetApplicationsRequest request = + Records.newRecord(GetApplicationsRequest.class); + request.setApplicationTypes(applicationTypes); + return request; + } + + /** + * Get the application types to filter applications on + * + * @return Set of Application Types to filter on + */ + @Public + @Stable + public abstract Set getApplicationTypes(); + + /** + * Set the application types to filter applications on + * + * @param applicationTypes + * A Set of Application Types to filter on. + * If not defined, match all applications + */ + @Private + @Unstable + public abstract void + setApplicationTypes(Set applicationTypes); +} diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/GetApplicationsResponse.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/GetApplicationsResponse.java new file mode 100644 index 0000000..45e8fdf --- /dev/null +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/GetApplicationsResponse.java @@ -0,0 +1,66 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.yarn.api.protocolrecords; + +import java.util.List; + +import org.apache.hadoop.classification.InterfaceAudience.Private; +import org.apache.hadoop.classification.InterfaceAudience.Public; +import org.apache.hadoop.classification.InterfaceStability.Stable; +import org.apache.hadoop.classification.InterfaceStability.Unstable; +import org.apache.hadoop.yarn.api.ApplicationClientProtocol; +import org.apache.hadoop.yarn.api.records.ApplicationReport; +import org.apache.hadoop.yarn.util.Records; + +/** + *

The response sent by the ResourceManager to a client + * requesting an {@link ApplicationReport} for applications.

+ * + *

The ApplicationReport for each application includes details + * such as user, queue, name, host on which the ApplicationMaster + * is running, RPC port, tracking URL, diagnostics, start time etc.

+ * + * @see ApplicationReport + * @see ApplicationClientProtocol#getApplications(GetApplicationsRequest) + */ +@Public +@Stable +public abstract class GetApplicationsResponse { + @Private + @Unstable + public static GetApplicationsResponse newInstance( + List applications) { + GetApplicationsResponse response = + Records.newRecord(GetApplicationsResponse.class); + response.setApplicationList(applications); + return response; + } + + /** + * Get ApplicationReport for applications. + * @return ApplicationReport for applications + */ + @Public + @Stable + public abstract List getApplicationList(); + + @Private + @Unstable + public abstract void setApplicationList(List applications); +} diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/applicationclient_protocol.proto hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/applicationclient_protocol.proto index a145f15..3f76849 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/applicationclient_protocol.proto +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/applicationclient_protocol.proto @@ -36,7 +36,7 @@ service ApplicationClientProtocolService { rpc submitApplication (SubmitApplicationRequestProto) returns (SubmitApplicationResponseProto); rpc forceKillApplication (KillApplicationRequestProto) returns (KillApplicationResponseProto); rpc getClusterMetrics (GetClusterMetricsRequestProto) returns (GetClusterMetricsResponseProto); - rpc getAllApplications (GetAllApplicationsRequestProto) returns (GetAllApplicationsResponseProto); + rpc getApplications (GetApplicationsRequestProto) returns (GetApplicationsResponseProto); rpc getClusterNodes (GetClusterNodesRequestProto) returns (GetClusterNodesResponseProto); rpc getQueueInfo (GetQueueInfoRequestProto) returns (GetQueueInfoResponseProto); rpc getQueueUserAcls (GetQueueUserAclsInfoRequestProto) returns (GetQueueUserAclsInfoResponseProto); diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/yarn_service_protos.proto hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/yarn_service_protos.proto index c82f5cc..dad075a 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/yarn_service_protos.proto +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/yarn_service_protos.proto @@ -123,10 +123,11 @@ message GetClusterMetricsResponseProto { optional YarnClusterMetricsProto cluster_metrics = 1; } -message GetAllApplicationsRequestProto { +message GetApplicationsRequestProto { + repeated string application_type = 1; } -message GetAllApplicationsResponseProto { +message GetApplicationsResponseProto { repeated ApplicationReportProto applications = 1; } diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/YarnClient.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/YarnClient.java index f55c5f3..149842d 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/YarnClient.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/YarnClient.java @@ -21,6 +21,7 @@ import java.io.IOException; import java.net.InetSocketAddress; import java.util.List; +import java.util.Set; import org.apache.hadoop.classification.InterfaceAudience; import org.apache.hadoop.classification.InterfaceAudience.Private; @@ -174,8 +175,28 @@ public abstract ApplicationReport getApplicationReport(ApplicationId appId) * @throws YarnException * @throws IOException */ - public abstract List getApplicationList() throws YarnException, - IOException; + public abstract List getApplicationList() + throws YarnException, IOException; + + /** + *

+ * Get a report (ApplicationReport) of Applications + * matching the given application types in the cluster. + *

+ * + *

+ * If the user does not have VIEW_APP access for an application + * then the corresponding report will be filtered as described in + * {@link #getApplicationReport(ApplicationId)}. + *

+ * + * @param applicationTypes + * @return a list of reports of applications + * @throws YarnException + * @throws IOException + */ + public abstract List getApplicationList( + Set applicationTypes) throws YarnException, IOException; /** *

diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/impl/YarnClientImpl.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/impl/YarnClientImpl.java index 7a7affb..78a3142 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/impl/YarnClientImpl.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/impl/YarnClientImpl.java @@ -22,7 +22,9 @@ import java.net.InetSocketAddress; import java.util.ArrayList; import java.util.EnumSet; +import java.util.HashSet; import java.util.List; +import java.util.Set; import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; @@ -32,8 +34,8 @@ import org.apache.hadoop.io.Text; import org.apache.hadoop.ipc.RPC; import org.apache.hadoop.yarn.api.ApplicationClientProtocol; -import org.apache.hadoop.yarn.api.protocolrecords.GetAllApplicationsRequest; -import org.apache.hadoop.yarn.api.protocolrecords.GetAllApplicationsResponse; +import org.apache.hadoop.yarn.api.protocolrecords.GetApplicationsRequest; +import org.apache.hadoop.yarn.api.protocolrecords.GetApplicationsResponse; import org.apache.hadoop.yarn.api.protocolrecords.GetApplicationReportRequest; import org.apache.hadoop.yarn.api.protocolrecords.GetApplicationReportResponse; import org.apache.hadoop.yarn.api.protocolrecords.GetClusterMetricsRequest; @@ -206,11 +208,17 @@ public ApplicationReport getApplicationReport(ApplicationId appId) } @Override - public List getApplicationList() - throws YarnException, IOException { - GetAllApplicationsRequest request = - Records.newRecord(GetAllApplicationsRequest.class); - GetAllApplicationsResponse response = rmClient.getAllApplications(request); + public List getApplicationList() throws YarnException, + IOException { + return getApplicationList(new HashSet()); + } + + @Override + public List getApplicationList( + Set applicationTypes) throws YarnException, IOException { + GetApplicationsRequest request = + GetApplicationsRequest.newInstance(applicationTypes); + GetApplicationsResponse response = rmClient.getApplications(request); return response.getApplicationList(); } diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/cli/ApplicationCLI.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/cli/ApplicationCLI.java index 312aab2..3403475 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/cli/ApplicationCLI.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/cli/ApplicationCLI.java @@ -21,11 +21,14 @@ import java.io.IOException; import java.io.PrintWriter; import java.text.DecimalFormat; +import java.util.HashSet; import java.util.List; +import java.util.Set; import org.apache.commons.cli.CommandLine; import org.apache.commons.cli.GnuParser; import org.apache.commons.cli.HelpFormatter; +import org.apache.commons.cli.Option; import org.apache.commons.cli.Options; import org.apache.hadoop.classification.InterfaceAudience.Private; import org.apache.hadoop.classification.InterfaceStability.Unstable; @@ -35,6 +38,8 @@ import org.apache.hadoop.yarn.exceptions.YarnException; import org.apache.hadoop.yarn.util.ConverterUtils; +import com.google.common.annotations.VisibleForTesting; + @Private @Unstable public class ApplicationCLI extends YarnCLI { @@ -42,6 +47,8 @@ "%30s\t%20s\t%20s\t%10s\t%10s\t%18s\t%18s\t%15s\t%35s" + System.getProperty("line.separator"); + private static final String APP_TYPE_CMD = "appTypes"; + public static void main(String[] args) throws Exception { ApplicationCLI cli = new ApplicationCLI(); cli.setSysOutPrintStream(System.out); @@ -56,8 +63,19 @@ public int run(String[] args) throws Exception { Options opts = new Options(); opts.addOption(STATUS_CMD, true, "Prints the status of the application."); - opts.addOption(LIST_CMD, false, "Lists all the Applications from RM."); + opts.addOption(LIST_CMD, false, "List applications from the RM. " + + "Supports optional use of --appTypes to filter applications " + + "based on application type."); opts.addOption(KILL_CMD, true, "Kills the application."); + opts.addOption(HELP_CMD, false, "Displays help for all commands."); + Option appTypeOpt = new Option(APP_TYPE_CMD, true, + "Works with --list to filter applications based on their type."); + appTypeOpt.setValueSeparator(','); + appTypeOpt.setArgs(Option.UNLIMITED_VALUES); + appTypeOpt.setArgName("Comma-separated list of application types"); + opts.addOption(appTypeOpt); + opts.getOption(KILL_CMD).setArgName("Application ID"); + opts.getOption(STATUS_CMD).setArgName("Application ID"); CommandLine cliParser = new GnuParser().parse(opts, args); int exitCode = -1; @@ -68,13 +86,27 @@ public int run(String[] args) throws Exception { } printApplicationReport(cliParser.getOptionValue(STATUS_CMD)); } else if (cliParser.hasOption(LIST_CMD)) { - listAllApplications(); + Set appTypes = new HashSet(); + if(cliParser.hasOption(APP_TYPE_CMD)) { + String[] types = cliParser.getOptionValues(APP_TYPE_CMD); + if (types != null) { + for (String type : types) { + if (!type.trim().isEmpty()) { + appTypes.add(type.trim()); + } + } + } + } + listApplications(appTypes); } else if (cliParser.hasOption(KILL_CMD)) { if (args.length != 2) { printUsage(opts); return exitCode; } killApplication(cliParser.getOptionValue(KILL_CMD)); + } else if (cliParser.hasOption(HELP_CMD)) { + printUsage(opts); + return 0; } else { syserr.println("Invalid Command Usage : "); printUsage(opts); @@ -87,19 +119,24 @@ public int run(String[] args) throws Exception { * * @param opts */ - private void printUsage(Options opts) { + @VisibleForTesting + void printUsage(Options opts) { new HelpFormatter().printHelp("application", opts); } /** - * Lists all the applications present in the Resource Manager + * Lists the applications matching the given application Types + * present in the Resource Manager * + * @param appTypes * @throws YarnException * @throws IOException */ - private void listAllApplications() throws YarnException, IOException { + private void listApplications(Set appTypes) + throws YarnException, IOException { PrintWriter writer = new PrintWriter(sysout); - List appsReport = client.getApplicationList(); + List appsReport = + client.getApplicationList(appTypes); writer.println("Total Applications:" + appsReport.size()); writer.printf(APPLICATIONS_PATTERN, "Application-Id", diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/cli/YarnCLI.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/cli/YarnCLI.java index 5f86033..921c135 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/cli/YarnCLI.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/cli/YarnCLI.java @@ -33,6 +33,7 @@ public static final String STATUS_CMD = "status"; public static final String LIST_CMD = "list"; public static final String KILL_CMD = "kill"; + public static final String HELP_CMD = "help"; protected PrintStream sysout; protected PrintStream syserr; protected YarnClient client; diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/cli/TestYarnCLI.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/cli/TestYarnCLI.java index d1f52a4..14f9b5c 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/cli/TestYarnCLI.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/cli/TestYarnCLI.java @@ -32,7 +32,9 @@ import java.io.PrintWriter; import java.util.ArrayList; import java.util.Date; +import java.util.HashSet; import java.util.List; +import java.util.Set; import junit.framework.Assert; @@ -51,6 +53,8 @@ import org.junit.Before; import org.junit.Test; +import org.apache.commons.cli.Options; + public class TestYarnCLI { private YarnClient client = mock(YarnClient.class); @@ -115,10 +119,31 @@ public void testGetAllApplications() throws Exception { FinalApplicationStatus.SUCCEEDED, null, "N/A", 0.53789f, "YARN"); List applicationReports = new ArrayList(); applicationReports.add(newApplicationReport); - when(client.getApplicationList()).thenReturn(applicationReports); - int result = cli.run(new String[] { "-list" }); + + ApplicationId applicationId2 = ApplicationId.newInstance(1234, 6); + ApplicationReport newApplicationReport2 = ApplicationReport.newInstance( + applicationId2, ApplicationAttemptId.newInstance(applicationId2, 2), + "user2", "queue2", "appname2", "host2", 125, null, + YarnApplicationState.FINISHED, "diagnostics2", "url2", 2, 2, + FinalApplicationStatus.SUCCEEDED, null, "N/A", 0.63789f, "NON-YARN"); + applicationReports.add(newApplicationReport2); + + ApplicationId applicationId3 = ApplicationId.newInstance(1234, 7); + ApplicationReport newApplicationReport3 = ApplicationReport.newInstance( + applicationId3, ApplicationAttemptId.newInstance(applicationId3, 3), + "user3", "queue3", "appname3", "host3", 126, null, + YarnApplicationState.FINISHED, "diagnostics3", "url3", 3, 3, + FinalApplicationStatus.SUCCEEDED, null, "N/A", 0.73789f, "MAPREDUCE"); + applicationReports.add(newApplicationReport3); + + Set appType1 = new HashSet(); + appType1.add("YARN"); + + when(client.getApplicationList(appType1)).thenReturn( + getApplicationReports(applicationReports, appType1)); + int result = cli.run(new String[] { "-list", "-appTypes", "YARN" }); assertEquals(0, result); - verify(client).getApplicationList(); + verify(client).getApplicationList(appType1); ByteArrayOutputStream baos = new ByteArrayOutputStream(); PrintWriter pw = new PrintWriter(baos); @@ -137,6 +162,138 @@ public void testGetAllApplications() throws Exception { String appsReportStr = baos.toString("UTF-8"); Assert.assertEquals(appsReportStr, sysOutStream.toString()); verify(sysOut, times(1)).write(any(byte[].class), anyInt(), anyInt()); + + sysOutStream.reset(); + Set appType2 = new HashSet(); + appType2.add("YARN"); + appType2.add("FOO-YARN"); + when(client.getApplicationList(appType2)).thenReturn( + getApplicationReports(applicationReports, appType2)); + cli.run(new String[] { "-list", "-appTypes", "YARN , ,, ,FOO-YARN", + ",,,,, YARN,," }); + assertEquals(0, result); + verify(client).getApplicationList(appType2); + baos = new ByteArrayOutputStream(); + pw = new PrintWriter(baos); + pw.println("Total Applications:1"); + pw.print(" Application-Id\t Application-Name"); + pw.print("\t Application-Type"); + pw.print("\t User\t Queue\t State\t "); + pw.print("Final-State\t Progress"); + pw.println("\t Tracking-URL"); + pw.print(" application_1234_0005\t "); + pw.print("appname\t YARN\t user\t "); + pw.print("queue\t FINISHED\t "); + pw.print("SUCCEEDED\t 53.79%"); + pw.println("\t N/A"); + pw.close(); + appsReportStr = baos.toString("UTF-8"); + Assert.assertEquals(appsReportStr, sysOutStream.toString()); + verify(sysOut, times(2)).write(any(byte[].class), anyInt(), anyInt()); + + sysOutStream.reset(); + Set appType3 = new HashSet(); + appType3.add("YARN"); + appType3.add("NON-YARN"); + when(client.getApplicationList(appType3)).thenReturn( + getApplicationReports(applicationReports, appType3)); + + result = cli.run(new String[] { "-list", "-appTypes", "YARN,NON-YARN" }); + assertEquals(0, result); + verify(client).getApplicationList(appType3); + baos = new ByteArrayOutputStream(); + pw = new PrintWriter(baos); + pw.println("Total Applications:2"); + pw.print(" Application-Id\t Application-Name"); + pw.print("\t Application-Type"); + pw.print("\t User\t Queue\t State\t "); + pw.print("Final-State\t Progress"); + pw.println("\t Tracking-URL"); + pw.print(" application_1234_0005\t "); + pw.print("appname\t YARN\t user\t "); + pw.print("queue\t FINISHED\t "); + pw.print("SUCCEEDED\t 53.79%"); + pw.println("\t N/A"); + pw.print(" application_1234_0006\t "); + pw.print("appname2\t NON-YARN\t user2\t "); + pw.print("queue2\t FINISHED\t "); + pw.print("SUCCEEDED\t 63.79%"); + pw.println("\t N/A"); + pw.close(); + appsReportStr = baos.toString("UTF-8"); + Assert.assertEquals(appsReportStr, sysOutStream.toString()); + verify(sysOut, times(3)).write(any(byte[].class), anyInt(), anyInt()); + + sysOutStream.reset(); + Set appType4 = new HashSet(); + when(client.getApplicationList(appType4)).thenReturn( + getApplicationReports(applicationReports, appType4)); + result = cli.run(new String[] { "-list" }); + assertEquals(0, result); + verify(client).getApplicationList(appType4); + + baos = new ByteArrayOutputStream(); + pw = new PrintWriter(baos); + pw.println("Total Applications:3"); + pw.print(" Application-Id\t Application-Name"); + pw.print("\t Application-Type"); + pw.print("\t User\t Queue\t State\t "); + pw.print("Final-State\t Progress"); + pw.println("\t Tracking-URL"); + pw.print(" application_1234_0005\t "); + pw.print("appname\t YARN\t user\t "); + pw.print("queue\t FINISHED\t "); + pw.print("SUCCEEDED\t 53.79%"); + pw.println("\t N/A"); + pw.print(" application_1234_0006\t "); + pw.print("appname2\t NON-YARN\t user2\t "); + pw.print("queue2\t FINISHED\t "); + pw.print("SUCCEEDED\t 63.79%"); + pw.println("\t N/A"); + pw.print(" application_1234_0007\t "); + pw.print("appname3\t MAPREDUCE\t user3\t "); + pw.print("queue3\t FINISHED\t "); + pw.print("SUCCEEDED\t 73.79%"); + pw.println("\t N/A"); + pw.close(); + appsReportStr = baos.toString("UTF-8"); + Assert.assertEquals(appsReportStr, sysOutStream.toString()); + verify(sysOut, times(4)).write(any(byte[].class), anyInt(), anyInt()); + } + + private List getApplicationReports( + List applicationReports, + Set appTypes) { + + List appReports = new ArrayList(); + boolean bypassFilter = appTypes.isEmpty(); + + for (ApplicationReport appReport : applicationReports) { + if (!(bypassFilter || appTypes.contains( + appReport.getApplicationType()))) { + continue; + } + appReports.add(appReport); + } + return appReports; + } + + @Test (timeout = 10000) + public void testHelpCommand() throws Exception { + ApplicationCLI cli = createAndGetAppCLI(); + ApplicationCLI spyCli = spy(cli); + int result = spyCli.run(new String[] { "-help" }); + Assert.assertTrue(result == 0); + verify(spyCli).printUsage(any(Options.class)); + + ApplicationId applicationId = ApplicationId.newInstance(1234, 5); + result = + cli.run(new String[] { "-kill", applicationId.toString(), "args" }); + verify(spyCli).printUsage(any(Options.class)); + + NodeId nodeId = NodeId.newInstance("host0", 0); + result = cli.run(new String[] { "-status", nodeId.toString(), "args" }); + verify(spyCli).printUsage(any(Options.class)); } @Test diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/impl/pb/client/ApplicationClientProtocolPBClientImpl.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/impl/pb/client/ApplicationClientProtocolPBClientImpl.java index e6cb402..88352ea 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/impl/pb/client/ApplicationClientProtocolPBClientImpl.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/impl/pb/client/ApplicationClientProtocolPBClientImpl.java @@ -33,8 +33,8 @@ import org.apache.hadoop.yarn.api.ApplicationClientProtocolPB; import org.apache.hadoop.yarn.api.protocolrecords.CancelDelegationTokenRequest; import org.apache.hadoop.yarn.api.protocolrecords.CancelDelegationTokenResponse; -import org.apache.hadoop.yarn.api.protocolrecords.GetAllApplicationsRequest; -import org.apache.hadoop.yarn.api.protocolrecords.GetAllApplicationsResponse; +import org.apache.hadoop.yarn.api.protocolrecords.GetApplicationsRequest; +import org.apache.hadoop.yarn.api.protocolrecords.GetApplicationsResponse; import org.apache.hadoop.yarn.api.protocolrecords.GetApplicationReportRequest; import org.apache.hadoop.yarn.api.protocolrecords.GetApplicationReportResponse; import org.apache.hadoop.yarn.api.protocolrecords.GetClusterMetricsRequest; @@ -57,8 +57,8 @@ import org.apache.hadoop.yarn.api.protocolrecords.SubmitApplicationResponse; import org.apache.hadoop.yarn.api.protocolrecords.impl.pb.CancelDelegationTokenRequestPBImpl; import org.apache.hadoop.yarn.api.protocolrecords.impl.pb.CancelDelegationTokenResponsePBImpl; -import org.apache.hadoop.yarn.api.protocolrecords.impl.pb.GetAllApplicationsRequestPBImpl; -import org.apache.hadoop.yarn.api.protocolrecords.impl.pb.GetAllApplicationsResponsePBImpl; +import org.apache.hadoop.yarn.api.protocolrecords.impl.pb.GetApplicationsRequestPBImpl; +import org.apache.hadoop.yarn.api.protocolrecords.impl.pb.GetApplicationsResponsePBImpl; import org.apache.hadoop.yarn.api.protocolrecords.impl.pb.GetApplicationReportRequestPBImpl; import org.apache.hadoop.yarn.api.protocolrecords.impl.pb.GetApplicationReportResponsePBImpl; import org.apache.hadoop.yarn.api.protocolrecords.impl.pb.GetClusterMetricsRequestPBImpl; @@ -81,7 +81,7 @@ import org.apache.hadoop.yarn.api.protocolrecords.impl.pb.SubmitApplicationResponsePBImpl; import org.apache.hadoop.yarn.exceptions.YarnException; import org.apache.hadoop.yarn.ipc.RPCUtil; -import org.apache.hadoop.yarn.proto.YarnServiceProtos.GetAllApplicationsRequestProto; +import org.apache.hadoop.yarn.proto.YarnServiceProtos.GetApplicationsRequestProto; import org.apache.hadoop.yarn.proto.YarnServiceProtos.GetApplicationReportRequestProto; import org.apache.hadoop.yarn.proto.YarnServiceProtos.GetClusterMetricsRequestProto; import org.apache.hadoop.yarn.proto.YarnServiceProtos.GetClusterNodesRequestProto; @@ -188,13 +188,13 @@ public SubmitApplicationResponse submitApplication( } @Override - public GetAllApplicationsResponse getAllApplications( - GetAllApplicationsRequest request) throws YarnException, + public GetApplicationsResponse getApplications( + GetApplicationsRequest request) throws YarnException, IOException { - GetAllApplicationsRequestProto requestProto = - ((GetAllApplicationsRequestPBImpl) request).getProto(); + GetApplicationsRequestProto requestProto = + ((GetApplicationsRequestPBImpl) request).getProto(); try { - return new GetAllApplicationsResponsePBImpl(proxy.getAllApplications( + return new GetApplicationsResponsePBImpl(proxy.getApplications( null, requestProto)); } catch (ServiceException e) { RPCUtil.unwrapAndThrowException(e); diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/impl/pb/service/ApplicationClientProtocolPBServiceImpl.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/impl/pb/service/ApplicationClientProtocolPBServiceImpl.java index d092edf..b38819d 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/impl/pb/service/ApplicationClientProtocolPBServiceImpl.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/impl/pb/service/ApplicationClientProtocolPBServiceImpl.java @@ -30,7 +30,7 @@ import org.apache.hadoop.yarn.api.ApplicationClientProtocol; import org.apache.hadoop.yarn.api.ApplicationClientProtocolPB; import org.apache.hadoop.yarn.api.protocolrecords.CancelDelegationTokenResponse; -import org.apache.hadoop.yarn.api.protocolrecords.GetAllApplicationsResponse; +import org.apache.hadoop.yarn.api.protocolrecords.GetApplicationsResponse; import org.apache.hadoop.yarn.api.protocolrecords.GetApplicationReportResponse; import org.apache.hadoop.yarn.api.protocolrecords.GetClusterMetricsResponse; import org.apache.hadoop.yarn.api.protocolrecords.GetClusterNodesResponse; @@ -43,8 +43,8 @@ import org.apache.hadoop.yarn.api.protocolrecords.SubmitApplicationResponse; import org.apache.hadoop.yarn.api.protocolrecords.impl.pb.CancelDelegationTokenRequestPBImpl; import org.apache.hadoop.yarn.api.protocolrecords.impl.pb.CancelDelegationTokenResponsePBImpl; -import org.apache.hadoop.yarn.api.protocolrecords.impl.pb.GetAllApplicationsRequestPBImpl; -import org.apache.hadoop.yarn.api.protocolrecords.impl.pb.GetAllApplicationsResponsePBImpl; +import org.apache.hadoop.yarn.api.protocolrecords.impl.pb.GetApplicationsRequestPBImpl; +import org.apache.hadoop.yarn.api.protocolrecords.impl.pb.GetApplicationsResponsePBImpl; import org.apache.hadoop.yarn.api.protocolrecords.impl.pb.GetApplicationReportRequestPBImpl; import org.apache.hadoop.yarn.api.protocolrecords.impl.pb.GetApplicationReportResponsePBImpl; import org.apache.hadoop.yarn.api.protocolrecords.impl.pb.GetClusterMetricsRequestPBImpl; @@ -66,8 +66,8 @@ import org.apache.hadoop.yarn.api.protocolrecords.impl.pb.SubmitApplicationRequestPBImpl; import org.apache.hadoop.yarn.api.protocolrecords.impl.pb.SubmitApplicationResponsePBImpl; import org.apache.hadoop.yarn.exceptions.YarnException; -import org.apache.hadoop.yarn.proto.YarnServiceProtos.GetAllApplicationsRequestProto; -import org.apache.hadoop.yarn.proto.YarnServiceProtos.GetAllApplicationsResponseProto; +import org.apache.hadoop.yarn.proto.YarnServiceProtos.GetApplicationsRequestProto; +import org.apache.hadoop.yarn.proto.YarnServiceProtos.GetApplicationsResponseProto; import org.apache.hadoop.yarn.proto.YarnServiceProtos.GetApplicationReportRequestProto; import org.apache.hadoop.yarn.proto.YarnServiceProtos.GetApplicationReportResponseProto; import org.apache.hadoop.yarn.proto.YarnServiceProtos.GetClusterMetricsRequestProto; @@ -170,14 +170,14 @@ public SubmitApplicationResponseProto submitApplication(RpcController arg0, } @Override - public GetAllApplicationsResponseProto getAllApplications( - RpcController controller, GetAllApplicationsRequestProto proto) + public GetApplicationsResponseProto getApplications( + RpcController controller, GetApplicationsRequestProto proto) throws ServiceException { - GetAllApplicationsRequestPBImpl request = - new GetAllApplicationsRequestPBImpl(proto); + GetApplicationsRequestPBImpl request = + new GetApplicationsRequestPBImpl(proto); try { - GetAllApplicationsResponse response = real.getAllApplications(request); - return ((GetAllApplicationsResponsePBImpl)response).getProto(); + GetApplicationsResponse response = real.getApplications(request); + return ((GetApplicationsResponsePBImpl)response).getProto(); } catch (YarnException e) { throw new ServiceException(e); } catch (IOException e) { diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/impl/pb/GetAllApplicationsRequestPBImpl.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/impl/pb/GetAllApplicationsRequestPBImpl.java deleted file mode 100644 index 24b16d0..0000000 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/impl/pb/GetAllApplicationsRequestPBImpl.java +++ /dev/null @@ -1,67 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.hadoop.yarn.api.protocolrecords.impl.pb; - -import org.apache.hadoop.classification.InterfaceAudience.Private; -import org.apache.hadoop.classification.InterfaceStability.Unstable; -import org.apache.hadoop.yarn.api.protocolrecords.GetAllApplicationsRequest; -import org.apache.hadoop.yarn.proto.YarnServiceProtos.GetAllApplicationsRequestProto; - -@Private -@Unstable -public class GetAllApplicationsRequestPBImpl extends GetAllApplicationsRequest { - GetAllApplicationsRequestProto proto = GetAllApplicationsRequestProto.getDefaultInstance(); - GetAllApplicationsRequestProto.Builder builder = null; - boolean viaProto = false; - - public GetAllApplicationsRequestPBImpl() { - builder = GetAllApplicationsRequestProto.newBuilder(); - } - - public GetAllApplicationsRequestPBImpl(GetAllApplicationsRequestProto proto) { - this.proto = proto; - viaProto = true; - } - - public GetAllApplicationsRequestProto getProto() { - proto = viaProto ? proto : builder.build(); - viaProto = true; - return proto; - } - - @Override - public int hashCode() { - return getProto().hashCode(); - } - - @Override - public boolean equals(Object other) { - if (other == null) - return false; - if (other.getClass().isAssignableFrom(this.getClass())) { - return this.getProto().equals(this.getClass().cast(other).getProto()); - } - return false; - } - - @Override - public String toString() { - return getProto().toString().replaceAll("\\n", ", ").replaceAll("\\s+", " "); - } -} diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/impl/pb/GetAllApplicationsResponsePBImpl.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/impl/pb/GetAllApplicationsResponsePBImpl.java deleted file mode 100644 index a26dccd..0000000 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/impl/pb/GetAllApplicationsResponsePBImpl.java +++ /dev/null @@ -1,174 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.hadoop.yarn.api.protocolrecords.impl.pb; - -import java.util.ArrayList; -import java.util.Iterator; -import java.util.List; - -import org.apache.hadoop.classification.InterfaceAudience.Private; -import org.apache.hadoop.classification.InterfaceStability.Unstable; -import org.apache.hadoop.yarn.api.protocolrecords.GetAllApplicationsResponse; -import org.apache.hadoop.yarn.api.records.ApplicationReport; -import org.apache.hadoop.yarn.api.records.impl.pb.ApplicationReportPBImpl; -import org.apache.hadoop.yarn.proto.YarnProtos.ApplicationReportProto; -import org.apache.hadoop.yarn.proto.YarnServiceProtos.GetAllApplicationsResponseProto; -import org.apache.hadoop.yarn.proto.YarnServiceProtos.GetAllApplicationsResponseProtoOrBuilder; - -@Private -@Unstable -public class GetAllApplicationsResponsePBImpl -extends GetAllApplicationsResponse { - - GetAllApplicationsResponseProto proto = - GetAllApplicationsResponseProto.getDefaultInstance(); - GetAllApplicationsResponseProto.Builder builder = null; - boolean viaProto = false; - - List applicationList; - - public GetAllApplicationsResponsePBImpl() { - builder = GetAllApplicationsResponseProto.newBuilder(); - } - - public GetAllApplicationsResponsePBImpl(GetAllApplicationsResponseProto proto) { - this.proto = proto; - viaProto = true; - } - - @Override - public List getApplicationList() { - initLocalApplicationsList(); - return this.applicationList; - } - - @Override - public void setApplicationList(List applications) { - maybeInitBuilder(); - if (applications == null) - builder.clearApplications(); - this.applicationList = applications; - } - - public GetAllApplicationsResponseProto getProto() { - mergeLocalToProto(); - proto = viaProto ? proto : builder.build(); - viaProto = true; - return proto; - } - - @Override - public int hashCode() { - return getProto().hashCode(); - } - - @Override - public boolean equals(Object other) { - if (other == null) - return false; - if (other.getClass().isAssignableFrom(this.getClass())) { - return this.getProto().equals(this.getClass().cast(other).getProto()); - } - return false; - } - - @Override - public String toString() { - return getProto().toString().replaceAll("\\n", ", ").replaceAll("\\s+", " "); - } - - private void mergeLocalToBuilder() { - if (this.applicationList != null) { - addLocalApplicationsToProto(); - } - } - - private void mergeLocalToProto() { - if (viaProto) - maybeInitBuilder(); - mergeLocalToBuilder(); - proto = builder.build(); - viaProto = true; - } - - private void maybeInitBuilder() { - if (viaProto || builder == null) { - builder = GetAllApplicationsResponseProto.newBuilder(proto); - } - viaProto = false; - } - - // Once this is called. containerList will never be null - until a getProto - // is called. - private void initLocalApplicationsList() { - if (this.applicationList != null) { - return; - } - GetAllApplicationsResponseProtoOrBuilder p = viaProto ? proto : builder; - List list = p.getApplicationsList(); - applicationList = new ArrayList(); - - for (ApplicationReportProto a : list) { - applicationList.add(convertFromProtoFormat(a)); - } - } - - private void addLocalApplicationsToProto() { - maybeInitBuilder(); - builder.clearApplications(); - if (applicationList == null) - return; - Iterable iterable = new Iterable() { - @Override - public Iterator iterator() { - return new Iterator() { - - Iterator iter = applicationList.iterator(); - - @Override - public boolean hasNext() { - return iter.hasNext(); - } - - @Override - public ApplicationReportProto next() { - return convertToProtoFormat(iter.next()); - } - - @Override - public void remove() { - throw new UnsupportedOperationException(); - - } - }; - - } - }; - builder.addAllApplications(iterable); - } - - private ApplicationReportPBImpl convertFromProtoFormat(ApplicationReportProto p) { - return new ApplicationReportPBImpl(p); - } - - private ApplicationReportProto convertToProtoFormat(ApplicationReport t) { - return ((ApplicationReportPBImpl)t).getProto(); - } - -} diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/impl/pb/GetApplicationsRequestPBImpl.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/impl/pb/GetApplicationsRequestPBImpl.java new file mode 100644 index 0000000..2f070f0 --- /dev/null +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/impl/pb/GetApplicationsRequestPBImpl.java @@ -0,0 +1,131 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.yarn.api.protocolrecords.impl.pb; + +import java.util.HashSet; +import java.util.List; +import java.util.Set; + +import org.apache.hadoop.classification.InterfaceAudience.Private; +import org.apache.hadoop.classification.InterfaceStability.Unstable; +import org.apache.hadoop.yarn.api.protocolrecords.GetApplicationsRequest; +import org.apache.hadoop.yarn.proto.YarnServiceProtos.GetApplicationsRequestProto; +import org.apache.hadoop.yarn.proto.YarnServiceProtos.GetApplicationsRequestProtoOrBuilder; + +@Private +@Unstable +public class GetApplicationsRequestPBImpl extends GetApplicationsRequest { + GetApplicationsRequestProto proto = GetApplicationsRequestProto.getDefaultInstance(); + GetApplicationsRequestProto.Builder builder = null; + boolean viaProto = false; + + Set applicationTypes = null; + + public GetApplicationsRequestPBImpl() { + builder = GetApplicationsRequestProto.newBuilder(); + } + + public GetApplicationsRequestPBImpl(GetApplicationsRequestProto proto) { + this.proto = proto; + viaProto = true; + } + + public GetApplicationsRequestProto getProto() { + mergeLocalToProto(); + proto = viaProto ? proto : builder.build(); + viaProto = true; + return proto; + } + + private void mergeLocalToProto() { + if (viaProto) + maybeInitBuilder(); + mergeLocalToBuilder(); + proto = builder.build(); + viaProto = true; + } + + private void mergeLocalToBuilder() { + if (this.applicationTypes != null) { + addApplicationTypesToProto(); + } + } + + private void addApplicationTypesToProto() { + maybeInitBuilder(); + builder.clearApplicationType(); + if (this.applicationTypes == null) + return; + builder.addAllApplicationType(applicationTypes); + } + + private void maybeInitBuilder() { + if (viaProto || builder == null) { + builder = GetApplicationsRequestProto.newBuilder(proto); + } + viaProto = false; + } + + private void initApplicationTypes() { + if (this.applicationTypes != null) { + return; + } + GetApplicationsRequestProtoOrBuilder p = viaProto ? proto : builder; + List appTypeList = p.getApplicationTypeList(); + this.applicationTypes = new HashSet(); + this.applicationTypes.addAll(appTypeList); + } + + @Override + public Set getApplicationTypes() { + initApplicationTypes(); + return this.applicationTypes; + } + + @Override + public void setApplicationTypes(Set applicationType) { + if (applicationType == null) { + this.applicationTypes.clear(); + return; + } + initApplicationTypes(); + this.applicationTypes.clear(); + this.applicationTypes.addAll(applicationType); + } + + @Override + public int hashCode() { + return getProto().hashCode(); + } + + @Override + public boolean equals(Object other) { + if (other == null) + return false; + if (other.getClass().isAssignableFrom(this.getClass())) { + return this.getProto().equals(this.getClass().cast(other).getProto()); + } + return false; + } + + @Override + public String toString() { + return getProto().toString().replaceAll("\\n", ", ").replaceAll("\\s+", " "); + } +} diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/impl/pb/GetApplicationsResponsePBImpl.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/impl/pb/GetApplicationsResponsePBImpl.java new file mode 100644 index 0000000..b0897c6 --- /dev/null +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/impl/pb/GetApplicationsResponsePBImpl.java @@ -0,0 +1,174 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.yarn.api.protocolrecords.impl.pb; + +import java.util.ArrayList; +import java.util.Iterator; +import java.util.List; + +import org.apache.hadoop.classification.InterfaceAudience.Private; +import org.apache.hadoop.classification.InterfaceStability.Unstable; +import org.apache.hadoop.yarn.api.protocolrecords.GetApplicationsResponse; +import org.apache.hadoop.yarn.api.records.ApplicationReport; +import org.apache.hadoop.yarn.api.records.impl.pb.ApplicationReportPBImpl; +import org.apache.hadoop.yarn.proto.YarnProtos.ApplicationReportProto; +import org.apache.hadoop.yarn.proto.YarnServiceProtos.GetApplicationsResponseProto; +import org.apache.hadoop.yarn.proto.YarnServiceProtos.GetApplicationsResponseProtoOrBuilder; + +@Private +@Unstable +public class GetApplicationsResponsePBImpl +extends GetApplicationsResponse { + + GetApplicationsResponseProto proto = + GetApplicationsResponseProto.getDefaultInstance(); + GetApplicationsResponseProto.Builder builder = null; + boolean viaProto = false; + + List applicationList; + + public GetApplicationsResponsePBImpl() { + builder = GetApplicationsResponseProto.newBuilder(); + } + + public GetApplicationsResponsePBImpl(GetApplicationsResponseProto proto) { + this.proto = proto; + viaProto = true; + } + + @Override + public List getApplicationList() { + initLocalApplicationsList(); + return this.applicationList; + } + + @Override + public void setApplicationList(List applications) { + maybeInitBuilder(); + if (applications == null) + builder.clearApplications(); + this.applicationList = applications; + } + + public GetApplicationsResponseProto getProto() { + mergeLocalToProto(); + proto = viaProto ? proto : builder.build(); + viaProto = true; + return proto; + } + + @Override + public int hashCode() { + return getProto().hashCode(); + } + + @Override + public boolean equals(Object other) { + if (other == null) + return false; + if (other.getClass().isAssignableFrom(this.getClass())) { + return this.getProto().equals(this.getClass().cast(other).getProto()); + } + return false; + } + + @Override + public String toString() { + return getProto().toString().replaceAll("\\n", ", ").replaceAll("\\s+", " "); + } + + private void mergeLocalToBuilder() { + if (this.applicationList != null) { + addLocalApplicationsToProto(); + } + } + + private void mergeLocalToProto() { + if (viaProto) + maybeInitBuilder(); + mergeLocalToBuilder(); + proto = builder.build(); + viaProto = true; + } + + private void maybeInitBuilder() { + if (viaProto || builder == null) { + builder = GetApplicationsResponseProto.newBuilder(proto); + } + viaProto = false; + } + + // Once this is called. containerList will never be null - until a getProto + // is called. + private void initLocalApplicationsList() { + if (this.applicationList != null) { + return; + } + GetApplicationsResponseProtoOrBuilder p = viaProto ? proto : builder; + List list = p.getApplicationsList(); + applicationList = new ArrayList(); + + for (ApplicationReportProto a : list) { + applicationList.add(convertFromProtoFormat(a)); + } + } + + private void addLocalApplicationsToProto() { + maybeInitBuilder(); + builder.clearApplications(); + if (applicationList == null) + return; + Iterable iterable = new Iterable() { + @Override + public Iterator iterator() { + return new Iterator() { + + Iterator iter = applicationList.iterator(); + + @Override + public boolean hasNext() { + return iter.hasNext(); + } + + @Override + public ApplicationReportProto next() { + return convertToProtoFormat(iter.next()); + } + + @Override + public void remove() { + throw new UnsupportedOperationException(); + + } + }; + + } + }; + builder.addAllApplications(iterable); + } + + private ApplicationReportPBImpl convertFromProtoFormat(ApplicationReportProto p) { + return new ApplicationReportPBImpl(p); + } + + private ApplicationReportProto convertToProtoFormat(ApplicationReport t) { + return ((ApplicationReportPBImpl)t).getProto(); + } + +} diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ClientRMService.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ClientRMService.java index 897d534..882ad51 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ClientRMService.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ClientRMService.java @@ -25,6 +25,7 @@ import java.util.Collection; import java.util.EnumSet; import java.util.List; +import java.util.Set; import java.util.concurrent.atomic.AtomicInteger; import org.apache.commons.logging.Log; @@ -42,8 +43,8 @@ import org.apache.hadoop.yarn.api.ApplicationClientProtocol; import org.apache.hadoop.yarn.api.protocolrecords.CancelDelegationTokenRequest; import org.apache.hadoop.yarn.api.protocolrecords.CancelDelegationTokenResponse; -import org.apache.hadoop.yarn.api.protocolrecords.GetAllApplicationsRequest; -import org.apache.hadoop.yarn.api.protocolrecords.GetAllApplicationsResponse; +import org.apache.hadoop.yarn.api.protocolrecords.GetApplicationsRequest; +import org.apache.hadoop.yarn.api.protocolrecords.GetApplicationsResponse; import org.apache.hadoop.yarn.api.protocolrecords.GetApplicationReportRequest; import org.apache.hadoop.yarn.api.protocolrecords.GetApplicationReportResponse; import org.apache.hadoop.yarn.api.protocolrecords.GetClusterMetricsRequest; @@ -391,8 +392,8 @@ public GetClusterMetricsResponse getClusterMetrics( } @Override - public GetAllApplicationsResponse getAllApplications( - GetAllApplicationsRequest request) throws YarnException { + public GetApplicationsResponse getApplications( + GetApplicationsRequest request) throws YarnException { UserGroupInformation callerUGI; try { @@ -402,15 +403,21 @@ public GetAllApplicationsResponse getAllApplications( throw RPCUtil.getRemoteException(ie); } + Set applicationTypes = request.getApplicationTypes(); + boolean bypassFilter = applicationTypes.isEmpty(); List reports = new ArrayList(); for (RMApp application : this.rmContext.getRMApps().values()) { + if (!(bypassFilter || applicationTypes.contains(application + .getApplicationType()))) { + continue; + } boolean allowAccess = checkAccess(callerUGI, application.getUser(), ApplicationAccessType.VIEW_APP, application.getApplicationId()); reports.add(application.createAndGetApplicationReport(allowAccess)); } - GetAllApplicationsResponse response = - recordFactory.newRecordInstance(GetAllApplicationsResponse.class); + GetApplicationsResponse response = + recordFactory.newRecordInstance(GetApplicationsResponse.class); response.setApplicationList(reports); return response; } diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestApplicationACLs.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestApplicationACLs.java index 1cf8045..8c28355 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestApplicationACLs.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestApplicationACLs.java @@ -34,7 +34,7 @@ import org.apache.hadoop.security.authorize.AccessControlList; import org.apache.hadoop.service.Service.STATE; import org.apache.hadoop.yarn.api.ApplicationClientProtocol; -import org.apache.hadoop.yarn.api.protocolrecords.GetAllApplicationsRequest; +import org.apache.hadoop.yarn.api.protocolrecords.GetApplicationsRequest; import org.apache.hadoop.yarn.api.protocolrecords.GetApplicationReportRequest; import org.apache.hadoop.yarn.api.protocolrecords.GetNewApplicationRequest; import org.apache.hadoop.yarn.api.protocolrecords.KillApplicationRequest; @@ -213,8 +213,8 @@ private void verifyOwnerAccess() throws Exception { // List apps as owner Assert.assertEquals("App view by owner should list the apps!!", 1, - rmClient.getAllApplications( - recordFactory.newRecordInstance(GetAllApplicationsRequest.class)) + rmClient.getApplications( + recordFactory.newRecordInstance(GetApplicationsRequest.class)) .getApplicationList().size()); // Kill app as owner @@ -244,8 +244,8 @@ private void verifySuperUserAccess() throws Exception { // List apps as superUser Assert.assertEquals("App view by super-user should list the apps!!", 2, - superUserClient.getAllApplications( - recordFactory.newRecordInstance(GetAllApplicationsRequest.class)) + superUserClient.getApplications( + recordFactory.newRecordInstance(GetApplicationsRequest.class)) .getApplicationList().size()); // Kill app as the superUser @@ -275,8 +275,8 @@ private void verifyFriendAccess() throws Exception { // List apps as friend Assert.assertEquals("App view by a friend should list the apps!!", 3, - friendClient.getAllApplications( - recordFactory.newRecordInstance(GetAllApplicationsRequest.class)) + friendClient.getApplications( + recordFactory.newRecordInstance(GetApplicationsRequest.class)) .getApplicationList().size()); // Kill app as the friend @@ -308,8 +308,8 @@ private void verifyEnemyAccess() throws Exception { // List apps as enemy List appReports = enemyRmClient - .getAllApplications(recordFactory - .newRecordInstance(GetAllApplicationsRequest.class)) + .getApplications(recordFactory + .newRecordInstance(GetApplicationsRequest.class)) .getApplicationList(); Assert.assertEquals("App view by enemy should list the apps!!", 4, appReports.size()); diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestClientRMService.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestClientRMService.java index 6910729..cd2e86c 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestClientRMService.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestClientRMService.java @@ -26,8 +26,10 @@ import java.io.IOException; import java.net.InetSocketAddress; import java.security.PrivilegedExceptionAction; +import java.util.HashSet; import java.util.EnumSet; import java.util.List; +import java.util.Set; import java.util.concurrent.BrokenBarrierException; import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.CyclicBarrier; @@ -42,6 +44,8 @@ import org.apache.hadoop.security.token.Token; import org.apache.hadoop.yarn.MockApps; import org.apache.hadoop.yarn.api.ApplicationClientProtocol; +import org.apache.hadoop.yarn.api.protocolrecords.GetApplicationsRequest; +import org.apache.hadoop.yarn.api.protocolrecords.GetApplicationsResponse; import org.apache.hadoop.yarn.api.protocolrecords.GetApplicationReportRequest; import org.apache.hadoop.yarn.api.protocolrecords.GetApplicationReportResponse; import org.apache.hadoop.yarn.api.protocolrecords.GetClusterNodesRequest; @@ -49,6 +53,7 @@ import org.apache.hadoop.yarn.api.protocolrecords.GetQueueInfoResponse; import org.apache.hadoop.yarn.api.protocolrecords.RenewDelegationTokenRequest; import org.apache.hadoop.yarn.api.protocolrecords.SubmitApplicationRequest; +import org.apache.hadoop.yarn.api.records.ApplicationAccessType; import org.apache.hadoop.yarn.api.records.ApplicationId; import org.apache.hadoop.yarn.api.records.ApplicationReport; import org.apache.hadoop.yarn.api.records.ApplicationSubmissionContext; @@ -88,6 +93,8 @@ private RecordFactory recordFactory = RecordFactoryProvider .getRecordFactory(null); + private String appType = "MockApp"; + private static RMDelegationTokenSecretManager dtsm; @BeforeClass @@ -292,11 +299,18 @@ public void testAppSubmit() throws Exception { new EventHandler() { public void handle(Event event) {} }); + ApplicationId appId1 = getApplicationId(100); + + ApplicationACLsManager mockAclsManager = mock(ApplicationACLsManager.class); + when( + mockAclsManager.checkAccess(UserGroupInformation.getCurrentUser(), + ApplicationAccessType.VIEW_APP, null, appId1)).thenReturn(true); ClientRMService rmService = - new ClientRMService(rmContext, yarnScheduler, appManager, null, null); + new ClientRMService(rmContext, yarnScheduler, appManager, + mockAclsManager, null); // without name and queue - ApplicationId appId1 = getApplicationId(100); + SubmitApplicationRequest submitRequest1 = mockSubmitAppRequest( appId1, null, null); try { @@ -317,6 +331,8 @@ public void handle(Event event) {} ApplicationId appId2 = getApplicationId(101); SubmitApplicationRequest submitRequest2 = mockSubmitAppRequest( appId2, name, queue); + submitRequest2.getApplicationSubmissionContext().setApplicationType( + "matchType"); try { rmService.submitApplication(submitRequest2); } catch (YarnException e) { @@ -335,6 +351,25 @@ public void handle(Event event) {} Assert.assertTrue("The thrown exception is not expected.", e.getMessage().contains("Cannot add a duplicate!")); } + + GetApplicationsRequest getAllAppsRequest = + GetApplicationsRequest.newInstance(new HashSet()); + GetApplicationsResponse getAllApplicationsResponse = + rmService.getApplications(getAllAppsRequest); + Assert.assertEquals(5, + getAllApplicationsResponse.getApplicationList().size()); + + Set appTypes = new HashSet(); + appTypes.add("matchType"); + + getAllAppsRequest = GetApplicationsRequest.newInstance(appTypes); + getAllApplicationsResponse = + rmService.getApplications(getAllAppsRequest); + Assert.assertEquals(1, + getAllApplicationsResponse.getApplicationList().size()); + Assert.assertEquals(appId2, + getAllApplicationsResponse.getApplicationList() + .get(0).getApplicationId()); } @Test(timeout=4000) @@ -416,6 +451,7 @@ private SubmitApplicationRequest mockSubmitAppRequest(ApplicationId appId, submissionContext.setQueue(queue); submissionContext.setApplicationId(appId); submissionContext.setResource(resource); + submissionContext.setApplicationType(appType); SubmitApplicationRequest submitRequest = recordFactory.newRecordInstance(SubmitApplicationRequest.class);