diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/DeSelectQuery.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/DeSelectQuery.java new file mode 100644 index 0000000..a3b5b59 --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/DeSelectQuery.java @@ -0,0 +1,99 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements.See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership.The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License.You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.yarn.server.resourcemanager.webapp; + +import java.util.Arrays; +import java.util.HashSet; +import java.util.Set; + +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.apache.hadoop.yarn.webapp.BadRequestException; + +/** + * DeSelectQuery make the /apps api more flexible. + * It can be used to strip off more fields if there's such use case in future. + * The process should follow two steps: + *
1. add a literal equivalent parameter XYZ in DeSelectParam + *
2. add a public boolean deSelectXYZ() in DeSelectQuery + */ +public class DeSelectQuery { + private static final Log LOG = + LogFactory.getLog(DeSelectQuery.class.getName()); + + private final Set deSelectParams; + + public DeSelectQuery() { + this.deSelectParams = new HashSet(); + } + + /** + * Initial DeSelect Query with unselected fields + * @param unselectedFields a set of unselected field. + */ + public void initQuery(Set unselectedFields) { + if (unselectedFields == null) { + return; + } + for (String field : unselectedFields) { + if (!field.trim().isEmpty()) { + String[] paramArray = field.split(","); + for (String param : paramArray) { + if (param != null && !param.trim().isEmpty()) { + try { + DeSelectParam.valueOf(param.trim()); + } catch (RuntimeException e) { + LOG.warn("Invalid deSelect string " + param.trim()); + DeSelectParam[] typeArray = + DeSelectParam.values(); + String allDeSelectTypes = Arrays.toString(typeArray); + throw new BadRequestException("Invalid deSelect string " + + param.trim() + " specified. It should be one of " + + allDeSelectTypes); + } + this.deSelectParams.add(DeSelectParam.valueOf(param.trim())); + } + } + } + } + } + + /** + * Determine resourceRequests deselected or not in query + * @return true if resourceRequests is deselected + */ + public boolean deSelectResourceRequests() { + return deSelectParams.contains(DeSelectParam.resourceRequests); + } + + /** + * Unselected parameter in query DeSelectQuery. + * Could be boost in future.
+ */ + public enum DeSelectParam { + + /** + * resourceRequests is the first supported parameter + * from YARN-6280. + *
It must be literal same with query parameter. + *
Usage: deselect="resourceRequests,.." + */ + resourceRequests, + + } +} \ No newline at end of file diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMWebServices.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMWebServices.java index bd0602b..cd6aeae 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMWebServices.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMWebServices.java @@ -432,7 +432,8 @@ public AppsInfo getApps(@Context HttpServletRequest hsr, @QueryParam("finishedTimeBegin") String finishBegin, @QueryParam("finishedTimeEnd") String finishEnd, @QueryParam("applicationTypes") Set applicationTypes, - @QueryParam("applicationTags") Set applicationTags) { + @QueryParam("applicationTags") Set applicationTags, + @QueryParam("deSelect") Set unselectedFields) { boolean checkCount = false; boolean checkStart = false; boolean checkEnd = false; @@ -588,8 +589,11 @@ public AppsInfo getApps(@Context HttpServletRequest hsr, } } - AppInfo app = new AppInfo(rm, rmapp, - hasAccess(rmapp, hsr), WebAppUtils.getHttpSchemePrefix(conf)); + DeSelectQuery deSelectQuery = new DeSelectQuery(); + deSelectQuery.initQuery(unselectedFields); + + AppInfo app = new AppInfo(rm, rmapp, hasAccess(rmapp, hsr), + WebAppUtils.getHttpSchemePrefix(conf), deSelectQuery); allApps.add(app); } return allApps; @@ -808,14 +812,20 @@ private static void countApp( @Produces({ MediaType.APPLICATION_JSON + "; " + JettyUtils.UTF_8, MediaType.APPLICATION_XML + "; " + JettyUtils.UTF_8 }) public AppInfo getApp(@Context HttpServletRequest hsr, - @PathParam("appid") String appId) { + @PathParam("appid") String appId, + @QueryParam("deSelect") Set unselectedFields) { init(); ApplicationId id = WebAppUtils.parseApplicationId(recordFactory, appId); RMApp app = rm.getRMContext().getRMApps().get(id); if (app == null) { throw new NotFoundException("app with id: " + appId + " not found"); } - return new AppInfo(rm, app, hasAccess(app, hsr), hsr.getScheme() + "://"); + + DeSelectQuery deSelectQuery = new DeSelectQuery(); + deSelectQuery.initQuery(unselectedFields); + + return new AppInfo(rm, app, hasAccess(app, hsr), + hsr.getScheme() + "://", deSelectQuery); } @GET diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/AppInfo.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/AppInfo.java index 10e627a..8b8d3eb 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/AppInfo.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/AppInfo.java @@ -44,6 +44,7 @@ import org.apache.hadoop.yarn.server.resourcemanager.scheduler.ResourceScheduler; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacityScheduler; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.common.fica.FiCaSchedulerApp; +import org.apache.hadoop.yarn.server.resourcemanager.webapp.DeSelectQuery; import org.apache.hadoop.yarn.util.Times; import org.apache.hadoop.yarn.webapp.util.WebAppUtils; @@ -121,9 +122,14 @@ public AppInfo() { } // JAXB needs this - @SuppressWarnings({ "rawtypes", "unchecked" }) public AppInfo(ResourceManager rm, RMApp app, Boolean hasAccess, String schemePrefix) { + this(rm, app, hasAccess, schemePrefix, new DeSelectQuery()); + } + + @SuppressWarnings({ "rawtypes", "unchecked" }) + public AppInfo(ResourceManager rm, RMApp app, Boolean hasAccess, + String schemePrefix, DeSelectQuery deSelectQuery) { this.schemePrefix = schemePrefix; if (app != null) { String trackingUrl = app.getTrackingUrl(); @@ -196,13 +202,19 @@ public AppInfo(ResourceManager rm, RMApp app, Boolean hasAccess, clusterUsagePercentage = resourceReport.getClusterUsagePercentage(); } - List resourceRequestsRaw = rm.getRMContext() - .getScheduler() - .getPendingResourceRequestsForAttempt(attempt.getAppAttemptId()); - - if (resourceRequestsRaw != null) { - for (ResourceRequest req : resourceRequestsRaw) { - resourceRequests.add(new ResourceRequestInfo(req)); + /* When the deSelect parameter contains "resourceRequests", + it skips returning massive ResourceRequest objects. + Default behavior is no skipping. (YARN-6280) + */ + if (!deSelectQuery.deSelectResourceRequests()) { + List resourceRequestsRaw = rm.getRMContext() + .getScheduler() + .getPendingResourceRequestsForAttempt(attempt.getAppAttemptId()); + + if (resourceRequestsRaw != null) { + for (ResourceRequest req : resourceRequestsRaw) { + resourceRequests.add(new ResourceRequestInfo(req)); + } } } } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServices.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServices.java index 6af2110..4f7ab54 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServices.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServices.java @@ -665,12 +665,12 @@ public void testAppsRace() throws Exception { // verify we don't get any apps when querying HttpServletRequest mockHsr = mock(HttpServletRequest.class); AppsInfo appsInfo = webSvc.getApps(mockHsr, null, emptySet, null, - null, null, null, null, null, null, null, emptySet, emptySet); + null, null, null, null, null, null, null, emptySet, emptySet, null); assertTrue(appsInfo.getApps().isEmpty()); // verify we don't get an NPE when specifying a final status query appsInfo = webSvc.getApps(mockHsr, null, emptySet, "FAILED", - null, null, null, null, null, null, null, emptySet, emptySet); + null, null, null, null, null, null, null, emptySet, emptySet, null); assertTrue(appsInfo.getApps().isEmpty()); } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesApps.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesApps.java index aab9bee..60cefa5 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesApps.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesApps.java @@ -1055,6 +1055,69 @@ public void testAppsQueryAppTypes() throws JSONException, Exception { } @Test + public void testAppsQueryWithInvaildDeselect() + throws JSONException, Exception { + try { + rm.start(); + MockNM amNodeManager = rm.registerNode("127.0.0.1:1234", 2048); + rm.submitApp(CONTAINER_MB); + amNodeManager.nodeHeartbeat(true); + WebResource r = resource(); + ClientResponse response = r.path("ws").path("v1").path("cluster") + .path("apps").queryParam("deSelect", "INVALIED_deselectParam") + .accept(MediaType.APPLICATION_JSON).get(ClientResponse.class); + assertResponseStatusCode(Status.BAD_REQUEST, response.getStatusInfo()); + assertEquals(MediaType.APPLICATION_JSON_TYPE + "; " + JettyUtils.UTF_8, + response.getType().toString()); + JSONObject msg = response.getEntity(JSONObject.class); + JSONObject exception = msg.getJSONObject("RemoteException"); + assertEquals("incorrect number of elements", 3, exception.length()); + String message = exception.getString("message"); + String type = exception.getString("exception"); + String classname = exception.getString("javaClassName"); + WebServicesTestUtils.checkStringContains("exception message", + "java.lang.Exception: Invalid deSelect string" + + " INVALIED_deselectParam " + "specified. It should be one of", + message); + WebServicesTestUtils.checkStringEqual("exception type", + "BadRequestException", type); + WebServicesTestUtils.checkStringEqual("exception classname", + "org.apache.hadoop.yarn.webapp.BadRequestException", classname); + } finally { + rm.stop(); + } + } + + @Test + public void testAppsQueryWithDeselect() + throws JSONException, Exception { + rm.start(); + MockNM amNodeManager = rm.registerNode("127.0.0.1:1234", 2048); + rm.submitApp(CONTAINER_MB); + amNodeManager.nodeHeartbeat(true); + WebResource r = resource(); + + MultivaluedMapImpl params = new MultivaluedMapImpl(); + params.add("deSelect", DeSelectQuery.DeSelectParam.resourceRequests.toString()); + ClientResponse response = r.path("ws").path("v1").path("cluster") + .path("apps").queryParams(params) + .accept(MediaType.APPLICATION_JSON).get(ClientResponse.class); + assertEquals(MediaType.APPLICATION_JSON_TYPE + "; " + JettyUtils.UTF_8, + response.getType().toString()); + + JSONObject json = response.getEntity(JSONObject.class); + assertEquals("incorrect number of elements", 1, json.length()); + JSONObject apps = json.getJSONObject("apps"); + assertEquals("incorrect number of elements", 1, apps.length()); + JSONArray array = apps.getJSONArray("app"); + assertEquals("incorrect number of elements", 1, array.length()); + JSONObject app = array.getJSONObject(0); + assertTrue("resource requests shouldn't exits", + !app.has("resourceRequests")); + rm.stop(); + } + + @Test public void testAppStatistics() throws JSONException, Exception { try { rm.start();