diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/JAXBContextResolver.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/JAXBContextResolver.java index 5f50ed1..2821f57 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/JAXBContextResolver.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/JAXBContextResolver.java @@ -34,12 +34,14 @@ import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.AppInfo; import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.AppAttemptInfo; import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.AppAttemptsInfo; +import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.AppsCountInfo; import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.AppsInfo; import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.CapacitySchedulerInfo; import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.CapacitySchedulerQueueInfo; import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.CapacitySchedulerQueueInfoList; import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.ClusterInfo; import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.ClusterMetricsInfo; +import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.CountItemInfo; import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.FifoSchedulerInfo; import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.NodeInfo; import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.NodesInfo; @@ -65,7 +67,8 @@ CapacitySchedulerInfo.class, ClusterMetricsInfo.class, SchedulerInfo.class, AppsInfo.class, NodesInfo.class, RemoteExceptionData.class, CapacitySchedulerQueueInfoList.class, - ResourceInfo.class, UsersInfo.class, UserInfo.class}; + ResourceInfo.class, UsersInfo.class, UserInfo.class, + AppsCountInfo.class, CountItemInfo.class}; public JAXBContextResolver() throws Exception { this.types = new HashSet(Arrays.asList(cTypes)); diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMWebServices.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMWebServices.java index a9b1523..b1e61f7 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMWebServices.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMWebServices.java @@ -21,7 +21,9 @@ import java.io.IOException; import java.util.Collection; import java.util.EnumSet; +import java.util.HashMap; import java.util.HashSet; +import java.util.Map; import java.util.Set; import java.util.concurrent.ConcurrentMap; @@ -57,10 +59,12 @@ import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.AppAttemptInfo; import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.AppAttemptsInfo; import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.AppInfo; +import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.AppsCountInfo; import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.AppsInfo; import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.CapacitySchedulerInfo; import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.ClusterInfo; import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.ClusterMetricsInfo; +import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.CountItemInfo; import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.FairSchedulerInfo; import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.FifoSchedulerInfo; import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.NodeInfo; @@ -384,6 +388,77 @@ public AppsInfo getApps(@Context HttpServletRequest hsr, } @GET + @Path("/appscount") + @Produces({ MediaType.APPLICATION_JSON, MediaType.APPLICATION_XML }) + public AppsCountInfo getAppsCount(@Context HttpServletRequest hsr, + @QueryParam("states") Set stateQueries, + @QueryParam("types") Set typeQueries) { + init(); + Set states = parseQueries(stateQueries); + Set types = parseQueries(typeQueries); + ConcurrentMap apps = rm.getRMContext().getRMApps(); + AppsCountInfo appsCount = new AppsCountInfo(); + Map> scoreboard + = new HashMap>(); + for (RMApp rmapp : apps.values()) { + String state = rmapp.getState().toString().toLowerCase(); + String type = rmapp.getApplicationType().trim().toLowerCase(); + if (states.isEmpty() && !types.isEmpty() && types.contains(type)) { + countApp(scoreboard, null, type); + } else if ( + !states.isEmpty() && states.contains(state) && types.isEmpty()) { + countApp(scoreboard, state, null); + } else if (!states.isEmpty() && states.contains(state) + && !types.isEmpty() && types.contains(type)) { + countApp(scoreboard, state, type); + } + } + for (Map.Entry> partScoreboard + : scoreboard.entrySet()) { + for (Map.Entry count + : partScoreboard.getValue().entrySet()) { + CountItemInfo countItem = new CountItemInfo( + (partScoreboard.getKey() == null ? "" : partScoreboard.getKey()), + (count.getKey() == null ? "" : count.getKey()), count.getValue()); + appsCount.add(countItem); + } + } + return appsCount; + } + + private static Set parseQueries(Set queries) { + Set params = new HashSet(); + if (!queries.isEmpty()) { + for (String query : queries) { + if (query != null && !query.trim().isEmpty()) { + String[] paramStrs = query.split(","); + for (String paramStr : paramStrs) { + if (paramStr != null && !paramStr.trim().isEmpty()) { + params.add(paramStr.trim().toLowerCase()); + } + } + } + } + } + return params; + } + + private static void countApp( + Map> scoreboard, String state, String type) { + Map partScoreboard = scoreboard.get(state); + if (partScoreboard == null) { + partScoreboard = new HashMap(); + scoreboard.put(state, partScoreboard); + } + Long count = partScoreboard.get(type); + if (count == null) { + partScoreboard.put(type, 1L); + } else { + partScoreboard.put(type, count + 1L); + } + } + + @GET @Path("/apps/{appid}") @Produces({ MediaType.APPLICATION_JSON, MediaType.APPLICATION_XML }) public AppInfo getApp(@Context HttpServletRequest hsr, diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/AppsCountInfo.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/AppsCountInfo.java new file mode 100644 index 0000000..40f982e --- /dev/null +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/AppsCountInfo.java @@ -0,0 +1,44 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.yarn.server.resourcemanager.webapp.dao; + +import java.util.ArrayList; + +import javax.xml.bind.annotation.XmlAccessType; +import javax.xml.bind.annotation.XmlAccessorType; +import javax.xml.bind.annotation.XmlRootElement; + +@XmlRootElement(name = "appsCount") +@XmlAccessorType(XmlAccessType.FIELD) +public class AppsCountInfo { + + protected ArrayList countItems + = new ArrayList(); + + public AppsCountInfo() { + } // JAXB needs this + + public void add(CountItemInfo countItem) { + countItems.add(countItem); + } + + public ArrayList getCountItems() { + return countItems; + } + +} diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/CountItemInfo.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/CountItemInfo.java new file mode 100644 index 0000000..f1d04aa --- /dev/null +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/CountItemInfo.java @@ -0,0 +1,53 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.yarn.server.resourcemanager.webapp.dao; + +import javax.xml.bind.annotation.XmlAccessType; +import javax.xml.bind.annotation.XmlAccessorType; +import javax.xml.bind.annotation.XmlRootElement; + +@XmlRootElement(name = "countItem") +@XmlAccessorType(XmlAccessType.FIELD) +public class CountItemInfo { + + protected String state; + protected String type; + protected long count; + + public CountItemInfo() { + } // JAXB needs this + + public CountItemInfo(String state, String type, long count) { + this.state = state; + this.type = type; + this.count = count; + } + + public String getState() { + return state; + } + + public String getType() { + return type; + } + + public long getCount() { + return count; + } + +} \ No newline at end of file diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesApps.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesApps.java index 52f72d8..6de5fe3 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesApps.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesApps.java @@ -823,6 +823,101 @@ public void testAppsQueryAppTypes() throws JSONException, Exception { } @Test + public void testAppsCount() throws JSONException, Exception { + rm.start(); + MockNM amNodeManager = rm.registerNode("127.0.0.1:1234", 4096); + Thread.sleep(1); + RMApp app1 = rm.submitApp(1024, "", UserGroupInformation.getCurrentUser() + .getShortUserName(), null, false, null, 2, null, "MAPREDUCE"); + amNodeManager.nodeHeartbeat(true); + // finish App + MockAM am = rm + .sendAMLaunched(app1.getCurrentAppAttempt().getAppAttemptId()); + am.registerAppAttempt(); + am.unregisterAppAttempt(); + amNodeManager.nodeHeartbeat(app1.getCurrentAppAttempt().getAppAttemptId(), + 1, ContainerState.COMPLETE); + + rm.submitApp(1024, "", UserGroupInformation.getCurrentUser() + .getShortUserName(), null, false, null, 2, null, "MAPREDUCE"); + rm.submitApp(1024, "", UserGroupInformation.getCurrentUser() + .getShortUserName(), null, false, null, 2, null, "OTHER"); + + WebResource r = resource(); + ClientResponse response = r.path("ws").path("v1").path("cluster") + .path("appscount").queryParam("types", "MAPREDUCE") + .accept(MediaType.APPLICATION_JSON).get(ClientResponse.class); + assertEquals(MediaType.APPLICATION_JSON_TYPE, response.getType()); + JSONObject json = response.getEntity(JSONObject.class); + assertEquals("incorrect number of elements", 1, json.length()); + JSONObject appsCount = json.getJSONObject("appsCount"); + assertEquals("incorrect number of elements", 1, appsCount.length()); + JSONArray countItems = appsCount.getJSONArray("countItems"); + assertEquals("incorrect number of elements", 1, countItems.length()); + assertEquals("2", countItems.getJSONObject(0).getString("count")); + + r = resource(); + response = r.path("ws").path("v1").path("cluster") + .path("appscount").queryParam("types", "MAPREDUCE,OTHER") + .accept(MediaType.APPLICATION_JSON).get(ClientResponse.class); + assertEquals(MediaType.APPLICATION_JSON_TYPE, response.getType()); + json = response.getEntity(JSONObject.class); + assertEquals("incorrect number of elements", 1, json.length()); + appsCount = json.getJSONObject("appsCount"); + assertEquals("incorrect number of elements", 1, appsCount.length()); + countItems = appsCount.getJSONArray("countItems"); + assertEquals("incorrect number of elements", 2, countItems.length()); + assertEquals("1", countItems.getJSONObject(0).getString("count")); + assertEquals("2", countItems.getJSONObject(1).getString("count")); + + r = resource(); + response = r.path("ws").path("v1").path("cluster") + .path("appscount").queryParam("states", RMAppState.FINISHED.toString()) + .accept(MediaType.APPLICATION_JSON).get(ClientResponse.class); + assertEquals(MediaType.APPLICATION_JSON_TYPE, response.getType()); + json = response.getEntity(JSONObject.class); + assertEquals("incorrect number of elements", 1, json.length()); + appsCount = json.getJSONObject("appsCount"); + assertEquals("incorrect number of elements", 1, appsCount.length()); + countItems = appsCount.getJSONArray("countItems"); + assertEquals("incorrect number of elements", 1, countItems.length()); + assertEquals("1", countItems.getJSONObject(0).getString("count")); + + r = resource(); + response = r.path("ws").path("v1").path("cluster") + .path("appscount").queryParam("states", RMAppState.FINISHED.toString() + + "," + RMAppState.ACCEPTED.toString()) + .accept(MediaType.APPLICATION_JSON).get(ClientResponse.class); + assertEquals(MediaType.APPLICATION_JSON_TYPE, response.getType()); + json = response.getEntity(JSONObject.class); + assertEquals("incorrect number of elements", 1, json.length()); + appsCount = json.getJSONObject("appsCount"); + assertEquals("incorrect number of elements", 1, appsCount.length()); + countItems = appsCount.getJSONArray("countItems"); + assertEquals("incorrect number of elements", 2, countItems.length()); + assertEquals("2", countItems.getJSONObject(0).getString("count")); + assertEquals("1", countItems.getJSONObject(1).getString("count")); + + r = resource(); + response = r.path("ws").path("v1").path("cluster") + .path("appscount").queryParam("states", RMAppState.FINISHED.toString() + + "," + RMAppState.ACCEPTED.toString()) + .queryParam("types", "MAPREDUCE,OTHER") + .accept(MediaType.APPLICATION_JSON).get(ClientResponse.class); + assertEquals(MediaType.APPLICATION_JSON_TYPE, response.getType()); + json = response.getEntity(JSONObject.class); + assertEquals("incorrect number of elements", 1, json.length()); + appsCount = json.getJSONObject("appsCount"); + assertEquals("incorrect number of elements", 1, appsCount.length()); + countItems = appsCount.getJSONArray("countItems"); + assertEquals("incorrect number of elements", 3, countItems.length()); + assertEquals("1", countItems.getJSONObject(0).getString("count")); + assertEquals("1", countItems.getJSONObject(1).getString("count")); + assertEquals("1", countItems.getJSONObject(2).getString("count")); + rm.stop(); + } + + @Test public void testSingleApp() throws JSONException, Exception { rm.start(); MockNM amNodeManager = rm.registerNode("127.0.0.1:1234", 2048);