diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/JAXBContextResolver.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/JAXBContextResolver.java index 5f50ed1..2821f57 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/JAXBContextResolver.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/JAXBContextResolver.java @@ -34,12 +34,14 @@ import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.AppInfo; import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.AppAttemptInfo; import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.AppAttemptsInfo; +import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.AppsCountInfo; import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.AppsInfo; import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.CapacitySchedulerInfo; import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.CapacitySchedulerQueueInfo; import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.CapacitySchedulerQueueInfoList; import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.ClusterInfo; import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.ClusterMetricsInfo; +import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.CountItemInfo; import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.FifoSchedulerInfo; import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.NodeInfo; import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.NodesInfo; @@ -65,7 +67,8 @@ CapacitySchedulerInfo.class, ClusterMetricsInfo.class, SchedulerInfo.class, AppsInfo.class, NodesInfo.class, RemoteExceptionData.class, CapacitySchedulerQueueInfoList.class, - ResourceInfo.class, UsersInfo.class, UserInfo.class}; + ResourceInfo.class, UsersInfo.class, UserInfo.class, + AppsCountInfo.class, CountItemInfo.class}; public JAXBContextResolver() throws Exception { this.types = new HashSet(Arrays.asList(cTypes)); diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMWebServices.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMWebServices.java index a9b1523..291b5be 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMWebServices.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMWebServices.java @@ -21,7 +21,9 @@ import java.io.IOException; import java.util.Collection; import java.util.EnumSet; +import java.util.HashMap; import java.util.HashSet; +import java.util.Map; import java.util.Set; import java.util.concurrent.ConcurrentMap; @@ -57,10 +59,12 @@ import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.AppAttemptInfo; import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.AppAttemptsInfo; import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.AppInfo; +import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.AppsCountInfo; import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.AppsInfo; import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.CapacitySchedulerInfo; import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.ClusterInfo; import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.ClusterMetricsInfo; +import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.CountItemInfo; import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.FairSchedulerInfo; import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.FifoSchedulerInfo; import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.NodeInfo; @@ -384,6 +388,115 @@ public AppsInfo getApps(@Context HttpServletRequest hsr, } @GET + @Path("/appscount") + @Produces({ MediaType.APPLICATION_JSON, MediaType.APPLICATION_XML }) + public AppsCountInfo getAppsCount(@Context HttpServletRequest hsr, + @QueryParam("states") Set stateQueries, + @QueryParam("types") Set typeQueries) { + init(); + + // build the scoreboard, converting state/type name to lowercase + Set states = parseQueries(stateQueries, true); + Set types = parseQueries(typeQueries, false); + if (states.isEmpty() && types.isEmpty()) { + return new AppsCountInfo(); + } + Map> scoreboard = buildScoreboard(states, types); + + // go through the apps in RM to count the numbers, ignoring the case of + // the state/type name + ConcurrentMap apps = rm.getRMContext().getRMApps(); + for (RMApp rmapp : apps.values()) { + String state = rmapp.getState().toString().toLowerCase(); + String type = rmapp.getApplicationType().trim().toLowerCase(); + if (states.isEmpty() && !types.isEmpty() && types.contains(type)) { + countApp(scoreboard, null, type); + } else if ( + !states.isEmpty() && states.contains(state) && types.isEmpty()) { + countApp(scoreboard, state, null); + } else if (!states.isEmpty() && states.contains(state) + && !types.isEmpty() && types.contains(type)) { + countApp(scoreboard, state, type); + } + } + + // fill the response object + AppsCountInfo appsCount = new AppsCountInfo(); + for (Map.Entry> partScoreboard + : scoreboard.entrySet()) { + for (Map.Entry count + : partScoreboard.getValue().entrySet()) { + CountItemInfo countItem = new CountItemInfo( + (partScoreboard.getKey() == null ? "" : partScoreboard.getKey()), + (count.getKey() == null ? "" : count.getKey()), count.getValue()); + appsCount.add(countItem); + } + } + return appsCount; + } + + private static Set parseQueries( + Set queries, boolean isState) { + Set params = new HashSet(); + if (!queries.isEmpty()) { + for (String query : queries) { + if (query != null && !query.trim().isEmpty()) { + String[] paramStrs = query.split(","); + for (String paramStr : paramStrs) { + if (paramStr != null && !paramStr.trim().isEmpty()) { + if (isState) { + try { + // enum string is in the uppercase + RMAppState.valueOf(paramStr.toUpperCase()); + } catch (RuntimeException e) { + throw new BadRequestException("Invalid application-state " + + paramStr + " specified. It should be one of" + + " NEW,ACCEPTED..(all-states).."); + } + } + params.add(paramStr.trim().toLowerCase()); + } + } + } + } + } + return params; + } + + private static Map> buildScoreboard( + Set states, Set types) { + Map> scoreboard + = new HashMap>(); + if (states.isEmpty()) { + Map partScoreboard = new HashMap(); + scoreboard.put(null, partScoreboard); + for (String type : types) { + partScoreboard.put(type, 0L); + } + } else { + for (String state : states) { + Map partScoreboard = new HashMap(); + scoreboard.put(state, partScoreboard); + if (types.isEmpty()) { + partScoreboard.put(null, 0L); + } else { + for (String type : types) { + partScoreboard.put(type, 0L); + } + } + } + } + return scoreboard; + } + + private static void countApp( + Map> scoreboard, String state, String type) { + Map partScoreboard = scoreboard.get(state); + Long count = partScoreboard.get(type); + partScoreboard.put(type, count + 1L); + } + + @GET @Path("/apps/{appid}") @Produces({ MediaType.APPLICATION_JSON, MediaType.APPLICATION_XML }) public AppInfo getApp(@Context HttpServletRequest hsr, diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/AppsCountInfo.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/AppsCountInfo.java new file mode 100644 index 0000000..b25fbad --- /dev/null +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/AppsCountInfo.java @@ -0,0 +1,44 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.yarn.server.resourcemanager.webapp.dao; + +import java.util.ArrayList; + +import javax.xml.bind.annotation.XmlAccessType; +import javax.xml.bind.annotation.XmlAccessorType; +import javax.xml.bind.annotation.XmlRootElement; + +@XmlRootElement(name = "appsCount") +@XmlAccessorType(XmlAccessType.FIELD) +public class AppsCountInfo { + + protected ArrayList countItem + = new ArrayList(); + + public AppsCountInfo() { + } // JAXB needs this + + public void add(CountItemInfo countItem) { + this.countItem.add(countItem); + } + + public ArrayList getCountItems() { + return countItem; + } + +} diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/CountItemInfo.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/CountItemInfo.java new file mode 100644 index 0000000..f1d04aa --- /dev/null +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/CountItemInfo.java @@ -0,0 +1,53 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.yarn.server.resourcemanager.webapp.dao; + +import javax.xml.bind.annotation.XmlAccessType; +import javax.xml.bind.annotation.XmlAccessorType; +import javax.xml.bind.annotation.XmlRootElement; + +@XmlRootElement(name = "countItem") +@XmlAccessorType(XmlAccessType.FIELD) +public class CountItemInfo { + + protected String state; + protected String type; + protected long count; + + public CountItemInfo() { + } // JAXB needs this + + public CountItemInfo(String state, String type, long count) { + this.state = state; + this.type = type; + this.count = count; + } + + public String getState() { + return state; + } + + public String getType() { + return type; + } + + public long getCount() { + return count; + } + +} \ No newline at end of file diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesApps.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesApps.java index 52f72d8..18bba89 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesApps.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesApps.java @@ -823,6 +823,146 @@ public void testAppsQueryAppTypes() throws JSONException, Exception { } @Test + public void testAppsCount() throws JSONException, Exception { + try { + rm.start(); + MockNM amNodeManager = rm.registerNode("127.0.0.1:1234", 4096); + Thread.sleep(1); + RMApp app1 = rm.submitApp(1024, "", UserGroupInformation.getCurrentUser() + .getShortUserName(), null, false, null, 2, null, "MAPREDUCE"); + amNodeManager.nodeHeartbeat(true); + // finish App + MockAM am = rm + .sendAMLaunched(app1.getCurrentAppAttempt().getAppAttemptId()); + am.registerAppAttempt(); + am.unregisterAppAttempt(); + amNodeManager.nodeHeartbeat(app1.getCurrentAppAttempt().getAppAttemptId(), + 1, ContainerState.COMPLETE); + + rm.submitApp(1024, "", UserGroupInformation.getCurrentUser() + .getShortUserName(), null, false, null, 2, null, "MAPREDUCE"); + rm.submitApp(1024, "", UserGroupInformation.getCurrentUser() + .getShortUserName(), null, false, null, 2, null, "OTHER"); + + // one type, zero state + WebResource r = resource(); + ClientResponse response = r.path("ws").path("v1").path("cluster") + .path("appscount") + .accept(MediaType.APPLICATION_JSON).get(ClientResponse.class); + assertEquals(MediaType.APPLICATION_JSON_TYPE, response.getType()); + JSONObject json = response.getEntity(JSONObject.class); + assertEquals("incorrect number of elements", 1, json.length()); + assertEquals("element should be null", JSONObject.NULL, + json.get("appsCount")); + + // one type, zero state + r = resource(); + response = r.path("ws").path("v1").path("cluster") + .path("appscount").queryParam("types", "MAPREDUCE") + .accept(MediaType.APPLICATION_JSON).get(ClientResponse.class); + assertEquals(MediaType.APPLICATION_JSON_TYPE, response.getType()); + json = response.getEntity(JSONObject.class); + assertEquals("incorrect number of elements", 1, json.length()); + JSONObject appsCount = json.getJSONObject("appsCount"); + assertEquals("incorrect number of elements", 1, appsCount.length()); + JSONArray countItems = appsCount.getJSONArray("countItem"); + assertEquals("incorrect number of elements", 1, countItems.length()); + assertEquals("2", countItems.getJSONObject(0).getString("count")); + + // two types, zero state + r = resource(); + response = r.path("ws").path("v1").path("cluster") + .path("appscount").queryParam("types", "MAPREDUCE,OTHER") + .accept(MediaType.APPLICATION_JSON).get(ClientResponse.class); + assertEquals(MediaType.APPLICATION_JSON_TYPE, response.getType()); + json = response.getEntity(JSONObject.class); + assertEquals("incorrect number of elements", 1, json.length()); + appsCount = json.getJSONObject("appsCount"); + assertEquals("incorrect number of elements", 1, appsCount.length()); + countItems = appsCount.getJSONArray("countItem"); + assertEquals("incorrect number of elements", 2, countItems.length()); + assertEquals("1", countItems.getJSONObject(0).getString("count")); + assertEquals("2", countItems.getJSONObject(1).getString("count")); + + // zero type, one state + r = resource(); + response = r.path("ws").path("v1").path("cluster") + .path("appscount") + .queryParam("states", RMAppState.FINISHED.toString()) + .accept(MediaType.APPLICATION_JSON).get(ClientResponse.class); + assertEquals(MediaType.APPLICATION_JSON_TYPE, response.getType()); + json = response.getEntity(JSONObject.class); + assertEquals("incorrect number of elements", 1, json.length()); + appsCount = json.getJSONObject("appsCount"); + assertEquals("incorrect number of elements", 1, appsCount.length()); + countItems = appsCount.getJSONArray("countItem"); + assertEquals("incorrect number of elements", 1, countItems.length()); + assertEquals("1", countItems.getJSONObject(0).getString("count")); + + // zero type, two states + r = resource(); + response = r.path("ws").path("v1").path("cluster") + .path("appscount") + .queryParam("states", RMAppState.FINISHED.toString() + + "," + RMAppState.ACCEPTED.toString()) + .accept(MediaType.APPLICATION_JSON).get(ClientResponse.class); + assertEquals(MediaType.APPLICATION_JSON_TYPE, response.getType()); + json = response.getEntity(JSONObject.class); + assertEquals("incorrect number of elements", 1, json.length()); + appsCount = json.getJSONObject("appsCount"); + assertEquals("incorrect number of elements", 1, appsCount.length()); + countItems = appsCount.getJSONArray("countItem"); + assertEquals("incorrect number of elements", 2, countItems.length()); + assertEquals("2", countItems.getJSONObject(0).getString("count")); + assertEquals("1", countItems.getJSONObject(1).getString("count")); + + // two types, two states + r = resource(); + response = r.path("ws").path("v1").path("cluster") + .path("appscount") + .queryParam("states", RMAppState.FINISHED.toString() + + "," + RMAppState.ACCEPTED.toString()) + .queryParam("types", "MAPREDUCE,OTHER") + .accept(MediaType.APPLICATION_JSON).get(ClientResponse.class); + assertEquals(MediaType.APPLICATION_JSON_TYPE, response.getType()); + json = response.getEntity(JSONObject.class); + assertEquals("incorrect number of elements", 1, json.length()); + appsCount = json.getJSONObject("appsCount"); + assertEquals("incorrect number of elements", 1, appsCount.length()); + countItems = appsCount.getJSONArray("countItem"); + assertEquals("incorrect number of elements", 4, countItems.length()); + assertEquals("1", countItems.getJSONObject(0).getString("count")); + assertEquals("1", countItems.getJSONObject(1).getString("count")); + assertEquals("0", countItems.getJSONObject(2).getString("count")); + assertEquals("1", countItems.getJSONObject(3).getString("count")); + + // invalid state + r = resource(); + response = r.path("ws").path("v1").path("cluster") + .path("appscount").queryParam("states", "wrong_state") + .accept(MediaType.APPLICATION_JSON).get(ClientResponse.class); + assertEquals(Status.BAD_REQUEST, response.getClientResponseStatus()); + assertEquals(MediaType.APPLICATION_JSON_TYPE, response.getType()); + json = response.getEntity(JSONObject.class); + assertEquals("incorrect number of elements", 1, json.length()); + JSONObject exception = json.getJSONObject("RemoteException"); + assertEquals("incorrect number of elements", 3, exception.length()); + String message = exception.getString("message"); + String type = exception.getString("exception"); + String className = exception.getString("javaClassName"); + WebServicesTestUtils.checkStringEqual("exception message", + "java.lang.Exception: Invalid application-state wrong_state specified." + + " It should be one of NEW,ACCEPTED..(all-states)..", message); + WebServicesTestUtils.checkStringEqual("exception type", + "BadRequestException", type); + WebServicesTestUtils.checkStringEqual("exception className", + "org.apache.hadoop.yarn.webapp.BadRequestException", className); + } finally { + rm.stop(); + } + } + + @Test public void testSingleApp() throws JSONException, Exception { rm.start(); MockNM amNodeManager = rm.registerNode("127.0.0.1:1234", 2048); diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/apt/ResourceManagerRest.apt.vm hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/apt/ResourceManagerRest.apt.vm index 5f05801..6218c56 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/apt/ResourceManagerRest.apt.vm +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/apt/ResourceManagerRest.apt.vm @@ -1268,6 +1268,139 @@ _01_000001 +---+ +* Cluster Applications Count API + + With the Applications Count API, you can obtain a collection of triples, each of which contains the application type, the application state and the number of applications in ResourceManager context with this state and type. When you run a GET operation on this resource, you obtain a collection of CountItem objects. + +** URI + +------ + * http:///ws/v1/cluster/appscount +------ + +** HTTP Operations Supported + +------ + * GET +------ + +** Query Parameters Required + + Two paramters can be specified. At least one parameters needs to be specified. Otherwise, the API will respond an empty response. + +------ + * states - states of the applications, specified as a comma-separated list + * types - types of the applications, specified as a comma-separated list. +------ + +** Elements of the (CountItems) object + + When you make a request for the list of count items, the information will be returned as a collection of countItem objects + +*-----------+----------------------------------------------------------------------+-------------------------------------+ +|| Item || Data Type || Description | +*-----------+----------------------------------------------------------------------+-------------------------------------+ +| countItem | array of countItem objects(JSON)/zero or more countItem objects(XML) | The collection of countItem objects | +*-----------+----------------------------------------------------------------------+-------------------------------------+ + +** Response Examples + + <> + + HTTP Request: + +------ + GET http:///ws/v1/cluster/appscount +------ + + Response Header: + ++---+ + HTTP/1.1 200 OK + Content-Type: application/json + Transfer-Encoding: chunked + Server: Jetty(6.1.26) ++---+ + + Response Body: + ++---+ +{ + "appsCount": + { + "countItem": + [ + { + "state" : "running", + "type" : "mapreduce", + "count" : 10 + }, + { + "state" : "finished", + "type" : "mapreduce", + "count" : 20 + }, + { + "state" : "running", + "type" : "yarn", + "count" : 0 + }, + { + "state" : "finished", + "type" : "yarn", + "count" : 5 + } + ] + } +} ++---+ + + <> + + HTTP Request: + +------ + GET http:///ws/v1/cluster/appscount + Accept: application/xml +------ + + Response Header: + ++---+ + HTTP/1.1 200 OK + Content-Type: application/xml + Content-Length: 2459 + Server: Jetty(6.1.26) ++---+ + + Response Body: + ++---+ + + + + running + mapreduce + 10 + + + finished + mapreduce + 20 + + + running + yarn + 0 + + + finished + yarn + 5 + + ++---+ + * Cluster {Application API} An application resource contains information about a particular application that was submitted to a cluster.