diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ClientRMService.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ClientRMService.java index be997534dd2..b4292038edf 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ClientRMService.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ClientRMService.java @@ -817,46 +817,7 @@ public GetApplicationsResponse getApplications(GetApplicationsRequest request) ApplicationsRequestScope scope = request.getScope(); final Map apps = rmContext.getRMApps(); - Iterator appsIter; - // If the query filters by queues, we can avoid considering apps outside - // of those queues by asking the scheduler for the apps in those queues. - if (queues != null && !queues.isEmpty()) { - // Construct an iterator over apps in given queues - // Collect list of lists to avoid copying all apps - final List> queueAppLists = - new ArrayList>(); - for (String queue : queues) { - List appsInQueue = scheduler.getAppsInQueue(queue); - if (appsInQueue != null && !appsInQueue.isEmpty()) { - queueAppLists.add(appsInQueue); - } - } - appsIter = new Iterator() { - Iterator> appListIter = queueAppLists.iterator(); - Iterator schedAppsIter; - - @Override - public boolean hasNext() { - // Because queueAppLists has no empty lists, hasNext is whether the - // current list hasNext or whether there are any remaining lists - return (schedAppsIter != null && schedAppsIter.hasNext()) - || appListIter.hasNext(); - } - @Override - public RMApp next() { - if (schedAppsIter == null || !schedAppsIter.hasNext()) { - schedAppsIter = appListIter.next().iterator(); - } - return apps.get(schedAppsIter.next().getApplicationId()); - } - @Override - public void remove() { - throw new UnsupportedOperationException("Remove not supported"); - } - }; - } else { - appsIter = apps.values().iterator(); - } + Iterator appsIter = apps.values().iterator(); List reports = new ArrayList(); while (appsIter.hasNext() && reports.size() < limit) { @@ -868,6 +829,12 @@ public void remove() { continue; } + if (queues != null && !queues.isEmpty()) { + if (!queues.contains(application.getQueue())) { + continue; + } + } + if (applicationTypes != null && !applicationTypes.isEmpty()) { String appTypeToMatch = StringUtils.toLowerCase(application.getApplicationType()); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesApps.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesApps.java index 15f94e1bb00..561125c7024 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesApps.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesApps.java @@ -20,17 +20,20 @@ import static org.apache.hadoop.yarn.webapp.WebServicesTestUtils.assertResponseStatusCode; import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertNotNull; import static org.junit.Assert.assertTrue; import static org.junit.Assert.fail; import java.io.StringReader; import java.util.ArrayList; import java.util.Collection; +import java.util.Map; import javax.ws.rs.core.MediaType; import javax.xml.parsers.DocumentBuilder; import javax.xml.parsers.DocumentBuilderFactory; +import com.google.common.collect.Maps; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.http.JettyUtils; import org.apache.hadoop.security.UserGroupInformation; @@ -104,6 +107,16 @@ protected void configureServlets() { Guice.createInjector(new WebServletModule())); } + private Map getAppsByApplicationIds(JSONArray array) throws JSONException { + Map apps = Maps.newHashMap(); + for (int i = 0; i < array.length(); i++) { + JSONObject app = array.getJSONObject(i); + String appId = (String) app.get("id"); + apps.put(appId, app); + } + return apps; + } + @Before @Override public void setUp() throws Exception { @@ -639,6 +652,80 @@ public void testAppsQueryQueue() throws JSONException, Exception { rm.stop(); } + @Test + public void testAppsQueryQueueAndStateFinished() throws Exception { + rm.start(); + MockNM amNodeManager = rm.registerNode("127.0.0.1:1234", 2048); + RMApp app1 = rm.submitApp(CONTAINER_MB); + RMApp app2 = rm.submitApp(CONTAINER_MB); + amNodeManager.nodeHeartbeat(true); + + finishApp(amNodeManager, app1); + finishApp(amNodeManager, app2); + rm.waitForState(app1.getApplicationId(), RMAppState.FINISHED); + rm.waitForState(app2.getApplicationId(), RMAppState.FINISHED); + + WebResource r = resource(); + + ClientResponse response = r.path("ws").path("v1").path("cluster") + .path("apps") + .queryParam("queue", "default") + .queryParam("state", YarnApplicationState.FINISHED.toString()) + .accept(MediaType.APPLICATION_JSON).get(ClientResponse.class); + assertEquals(MediaType.APPLICATION_JSON_TYPE + "; " + JettyUtils.UTF_8, + response.getType().toString()); + JSONObject json = response.getEntity(JSONObject.class); + assertEquals("incorrect number of elements", 1, json.length()); + JSONObject apps = json.getJSONObject("apps"); + assertEquals("incorrect number of elements", 1, apps.length()); + JSONArray array = apps.getJSONArray("app"); + + Map appJsons = getAppsByApplicationIds(array); + JSONObject finishedApp1Json = appJsons.get(app1.getApplicationId().toString()); + JSONObject finishedApp2Json = appJsons.get(app2.getApplicationId().toString()); + assertNotNull("Finished app1 should be in the result list!", finishedApp1Json); + assertNotNull("Finished app2 should be in the result list!", finishedApp2Json); + assertEquals("incorrect number of elements", 2, array.length()); + + rm.stop(); + } + + @Test + public void testAppsQueryQueue2() throws Exception { + rm.start(); + MockNM amNodeManager = rm.registerNode("127.0.0.1:1234", 2048); + RMApp finishedApp = rm.submitApp(CONTAINER_MB); + RMApp runningApp = rm.submitApp(CONTAINER_MB); + amNodeManager.nodeHeartbeat(true); + + finishApp(amNodeManager, finishedApp); + rm.waitForState(finishedApp.getApplicationId(), RMAppState.FINISHED); + + WebResource r = resource(); + + ClientResponse response = r.path("ws").path("v1").path("cluster") + .path("apps") + .queryParam("queue", "default") + .accept(MediaType.APPLICATION_JSON).get(ClientResponse.class); + assertEquals(MediaType.APPLICATION_JSON_TYPE + "; " + JettyUtils.UTF_8, + response.getType().toString()); + JSONObject json = response.getEntity(JSONObject.class); + assertEquals("incorrect number of elements", 1, json.length()); + JSONObject apps = json.getJSONObject("apps"); + assertEquals("incorrect number of elements", 1, apps.length()); + + JSONArray array = apps.getJSONArray("app"); + + Map appJsons = getAppsByApplicationIds(array); + JSONObject runningAppJson = appJsons.get(runningApp.getApplicationId().toString()); + JSONObject finishedAppJson = appJsons.get(finishedApp.getApplicationId().toString()); + assertNotNull("Running app should be in the result list!", runningAppJson); + assertNotNull("Finished app should be in the result list!", finishedAppJson); + assertEquals("incorrect number of elements", 2, array.length()); + + rm.stop(); + } + @Test public void testAppsQueryLimit() throws JSONException, Exception { rm.start(); @@ -766,13 +853,7 @@ public void testAppsQueryFinishBegin() throws JSONException, Exception { Thread.sleep(1); RMApp app1 = rm.submitApp(CONTAINER_MB); amNodeManager.nodeHeartbeat(true); - // finish App - MockAM am = rm - .sendAMLaunched(app1.getCurrentAppAttempt().getAppAttemptId()); - am.registerAppAttempt(); - am.unregisterAppAttempt(); - amNodeManager.nodeHeartbeat(app1.getCurrentAppAttempt().getAppAttemptId(), - 1, ContainerState.COMPLETE); + finishApp(amNodeManager, app1); rm.submitApp(CONTAINER_MB); rm.submitApp(CONTAINER_MB); @@ -791,6 +872,15 @@ public void testAppsQueryFinishBegin() throws JSONException, Exception { rm.stop(); } + private void finishApp(MockNM amNodeManager, RMApp app) throws Exception { + MockAM am = rm + .sendAMLaunched(app.getCurrentAppAttempt().getAppAttemptId()); + am.registerAppAttempt(); + am.unregisterAppAttempt(); + amNodeManager.nodeHeartbeat(app.getCurrentAppAttempt().getAppAttemptId(), + 1, ContainerState.COMPLETE); + } + @Test public void testAppsQueryFinishEnd() throws JSONException, Exception { rm.start(); @@ -798,12 +888,7 @@ public void testAppsQueryFinishEnd() throws JSONException, Exception { RMApp app1 = rm.submitApp(CONTAINER_MB); amNodeManager.nodeHeartbeat(true); // finish App - MockAM am = rm - .sendAMLaunched(app1.getCurrentAppAttempt().getAppAttemptId()); - am.registerAppAttempt(); - am.unregisterAppAttempt(); - amNodeManager.nodeHeartbeat(app1.getCurrentAppAttempt().getAppAttemptId(), - 1, ContainerState.COMPLETE); + finishApp(amNodeManager, app1); rm.submitApp(CONTAINER_MB); rm.submitApp(CONTAINER_MB); @@ -833,12 +918,7 @@ public void testAppsQueryFinishBeginEnd() throws JSONException, Exception { RMApp app1 = rm.submitApp(CONTAINER_MB); amNodeManager.nodeHeartbeat(true); // finish App - MockAM am = rm - .sendAMLaunched(app1.getCurrentAppAttempt().getAppAttemptId()); - am.registerAppAttempt(); - am.unregisterAppAttempt(); - amNodeManager.nodeHeartbeat(app1.getCurrentAppAttempt().getAppAttemptId(), - 1, ContainerState.COMPLETE); + finishApp(amNodeManager, app1); rm.submitApp(CONTAINER_MB); rm.submitApp(CONTAINER_MB); @@ -868,12 +948,7 @@ public void testAppsQueryAppTypes() throws JSONException, Exception { RMApp app1 = rm.submitApp(CONTAINER_MB); amNodeManager.nodeHeartbeat(true); // finish App - MockAM am = rm - .sendAMLaunched(app1.getCurrentAppAttempt().getAppAttemptId()); - am.registerAppAttempt(); - am.unregisterAppAttempt(); - amNodeManager.nodeHeartbeat(app1.getCurrentAppAttempt().getAppAttemptId(), - 1, ContainerState.COMPLETE); + finishApp(amNodeManager, app1); rm.submitApp(CONTAINER_MB, "", UserGroupInformation.getCurrentUser() .getShortUserName(), null, false, null, 2, null, "MAPREDUCE"); @@ -1203,13 +1278,7 @@ public void testAppStatistics() throws JSONException, Exception { RMApp app1 = rm.submitApp(CONTAINER_MB, "", UserGroupInformation.getCurrentUser() .getShortUserName(), null, false, null, 2, null, "MAPREDUCE"); amNodeManager.nodeHeartbeat(true); - // finish App - MockAM am = rm - .sendAMLaunched(app1.getCurrentAppAttempt().getAppAttemptId()); - am.registerAppAttempt(); - am.unregisterAppAttempt(); - amNodeManager.nodeHeartbeat(app1.getCurrentAppAttempt().getAppAttemptId(), - 1, ContainerState.COMPLETE); + finishApp(amNodeManager, app1); rm.submitApp(CONTAINER_MB, "", UserGroupInformation.getCurrentUser() .getShortUserName(), null, false, null, 2, null, "MAPREDUCE");