diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/DeSelectQuery.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/DeSelectQuery.java new file mode 100644 index 0000000..eeaca32 --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/DeSelectQuery.java @@ -0,0 +1,81 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements.See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership.The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License.You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.yarn.server.resourcemanager; + +import java.util.Arrays; +import java.util.HashSet; +import java.util.Set; + +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.apache.hadoop.yarn.webapp.BadRequestException; + +public class DeSelectQuery { + private static final Log LOG = + LogFactory.getLog(DeSelectQuery.class.getName()); + private Set deSelect; + + public DeSelectQuery() { + this.deSelect = new HashSet(); + } + + public DeSelectQuery(Set deSelect) { + this(); + setDeSelect(deSelect); + } + + private void setDeSelect(Set deSelect) { + if (deSelect != null && !deSelect.isEmpty()) { + for (String query : deSelect) { + if (query != null && !query.trim().isEmpty()) { + String[] paramStrs = query.split(","); + for (String paramStr : paramStrs) { + if (paramStr != null && !paramStr.trim().isEmpty()) { + try { + DeSelectParam.valueOf(paramStr.trim()); + } catch (RuntimeException e) { + LOG.info("Invalid deSelect string " + paramStr.trim()); + DeSelectParam[] typeArray = + DeSelectParam.values(); + String allDeSelectTypes = Arrays.toString(typeArray); + throw new BadRequestException("Invalid deSelect string " + + paramStr.trim() + " specified. It should be one of " + + allDeSelectTypes); + } + this.deSelect.add(DeSelectParam.valueOf(paramStr.trim())); + } + } + } + } + } + } + + public boolean hideResourceRequests() { + return deSelect.contains(DeSelectParam.resourceRequests); + } + + /** + * Enumeration of parameter of query DeSelect. + */ + public enum DeSelectParam { + /** add deDelect parameters here **/ + + /** resource requests. */ + resourceRequests, + } +} \ No newline at end of file diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMWebServices.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMWebServices.java index bd0602b..2c67185 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMWebServices.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMWebServices.java @@ -127,6 +127,7 @@ import org.apache.hadoop.yarn.factories.RecordFactory; import org.apache.hadoop.yarn.factory.providers.RecordFactoryProvider; import org.apache.hadoop.yarn.security.client.RMDelegationTokenIdentifier; +import org.apache.hadoop.yarn.server.resourcemanager.DeSelectQuery; import org.apache.hadoop.yarn.server.resourcemanager.RMAuditLogger; import org.apache.hadoop.yarn.server.resourcemanager.RMAuditLogger.AuditConstants; import org.apache.hadoop.yarn.server.resourcemanager.RMServerUtils; @@ -432,7 +433,8 @@ public AppsInfo getApps(@Context HttpServletRequest hsr, @QueryParam("finishedTimeBegin") String finishBegin, @QueryParam("finishedTimeEnd") String finishEnd, @QueryParam("applicationTypes") Set applicationTypes, - @QueryParam("applicationTags") Set applicationTags) { + @QueryParam("applicationTags") Set applicationTags, + @QueryParam("deSelect") Set deSelect) { boolean checkCount = false; boolean checkStart = false; boolean checkEnd = false; @@ -588,8 +590,10 @@ public AppsInfo getApps(@Context HttpServletRequest hsr, } } - AppInfo app = new AppInfo(rm, rmapp, - hasAccess(rmapp, hsr), WebAppUtils.getHttpSchemePrefix(conf)); + DeSelectQuery deSelectQuery = new DeSelectQuery(deSelect); + + AppInfo app = new AppInfo(rm, rmapp, hasAccess(rmapp, hsr), + WebAppUtils.getHttpSchemePrefix(conf), deSelectQuery); allApps.add(app); } return allApps; @@ -808,14 +812,19 @@ private static void countApp( @Produces({ MediaType.APPLICATION_JSON + "; " + JettyUtils.UTF_8, MediaType.APPLICATION_XML + "; " + JettyUtils.UTF_8 }) public AppInfo getApp(@Context HttpServletRequest hsr, - @PathParam("appid") String appId) { + @PathParam("appid") String appId, + @QueryParam("deSelect") Set deSelect) { init(); ApplicationId id = WebAppUtils.parseApplicationId(recordFactory, appId); RMApp app = rm.getRMContext().getRMApps().get(id); if (app == null) { throw new NotFoundException("app with id: " + appId + " not found"); } - return new AppInfo(rm, app, hasAccess(app, hsr), hsr.getScheme() + "://"); + + DeSelectQuery deSelectQuery = new DeSelectQuery(deSelect); + + return new AppInfo(rm, app, hasAccess(app, hsr), + hsr.getScheme() + "://", deSelectQuery); } @GET diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/AppInfo.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/AppInfo.java index 10e627a..564a1e5 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/AppInfo.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/AppInfo.java @@ -37,6 +37,7 @@ import org.apache.hadoop.yarn.api.records.Resource; import org.apache.hadoop.yarn.api.records.ResourceRequest; import org.apache.hadoop.yarn.api.records.YarnApplicationState; +import org.apache.hadoop.yarn.server.resourcemanager.DeSelectQuery; import org.apache.hadoop.yarn.server.resourcemanager.ResourceManager; import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMApp; import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMAppMetrics; @@ -121,9 +122,14 @@ public AppInfo() { } // JAXB needs this - @SuppressWarnings({ "rawtypes", "unchecked" }) public AppInfo(ResourceManager rm, RMApp app, Boolean hasAccess, String schemePrefix) { + this(rm, app, hasAccess, schemePrefix, new DeSelectQuery()); + } + + @SuppressWarnings({ "rawtypes", "unchecked" }) + public AppInfo(ResourceManager rm, RMApp app, Boolean hasAccess, + String schemePrefix, DeSelectQuery deSelect) { this.schemePrefix = schemePrefix; if (app != null) { String trackingUrl = app.getTrackingUrl(); @@ -196,13 +202,19 @@ public AppInfo(ResourceManager rm, RMApp app, Boolean hasAccess, clusterUsagePercentage = resourceReport.getClusterUsagePercentage(); } - List resourceRequestsRaw = rm.getRMContext() - .getScheduler() - .getPendingResourceRequestsForAttempt(attempt.getAppAttemptId()); - - if (resourceRequestsRaw != null) { - for (ResourceRequest req : resourceRequestsRaw) { - resourceRequests.add(new ResourceRequestInfo(req)); + /* When the deSelect parameter contains "resourceRequests", + it skips returning massive ResourceRequest objects. + Default behavior is no skipping. (YARN-6280) + */ + if (!deSelect.hideResourceRequests()) { + List resourceRequestsRaw = rm.getRMContext() + .getScheduler() + .getPendingResourceRequestsForAttempt(attempt.getAppAttemptId()); + + if (resourceRequestsRaw != null) { + for (ResourceRequest req : resourceRequestsRaw) { + resourceRequests.add(new ResourceRequestInfo(req)); + } } } } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServices.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServices.java index 6af2110..4f7ab54 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServices.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServices.java @@ -665,12 +665,12 @@ public void testAppsRace() throws Exception { // verify we don't get any apps when querying HttpServletRequest mockHsr = mock(HttpServletRequest.class); AppsInfo appsInfo = webSvc.getApps(mockHsr, null, emptySet, null, - null, null, null, null, null, null, null, emptySet, emptySet); + null, null, null, null, null, null, null, emptySet, emptySet, null); assertTrue(appsInfo.getApps().isEmpty()); // verify we don't get an NPE when specifying a final status query appsInfo = webSvc.getApps(mockHsr, null, emptySet, "FAILED", - null, null, null, null, null, null, null, emptySet, emptySet); + null, null, null, null, null, null, null, emptySet, emptySet, null); assertTrue(appsInfo.getApps().isEmpty()); } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesApps.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesApps.java index aab9bee..2dfc76e 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesApps.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesApps.java @@ -39,6 +39,7 @@ import org.apache.hadoop.yarn.api.records.ResourceRequest; import org.apache.hadoop.yarn.api.records.YarnApplicationState; import org.apache.hadoop.yarn.conf.YarnConfiguration; +import org.apache.hadoop.yarn.server.resourcemanager.DeSelectQuery; import org.apache.hadoop.yarn.server.resourcemanager.MockAM; import org.apache.hadoop.yarn.server.resourcemanager.MockNM; import org.apache.hadoop.yarn.server.resourcemanager.MockRM; @@ -1055,6 +1056,69 @@ public void testAppsQueryAppTypes() throws JSONException, Exception { } @Test + public void testAppsQueryWithInvaildDeselect() + throws JSONException, Exception { + try { + rm.start(); + MockNM amNodeManager = rm.registerNode("127.0.0.1:1234", 2048); + rm.submitApp(CONTAINER_MB); + amNodeManager.nodeHeartbeat(true); + WebResource r = resource(); + ClientResponse response = r.path("ws").path("v1").path("cluster") + .path("apps").queryParam("deSelect", "INVALIED_deselectParam") + .accept(MediaType.APPLICATION_JSON).get(ClientResponse.class); + assertResponseStatusCode(Status.BAD_REQUEST, response.getStatusInfo()); + assertEquals(MediaType.APPLICATION_JSON_TYPE + "; " + JettyUtils.UTF_8, + response.getType().toString()); + JSONObject msg = response.getEntity(JSONObject.class); + JSONObject exception = msg.getJSONObject("RemoteException"); + assertEquals("incorrect number of elements", 3, exception.length()); + String message = exception.getString("message"); + String type = exception.getString("exception"); + String classname = exception.getString("javaClassName"); + WebServicesTestUtils.checkStringContains("exception message", + "java.lang.Exception: Invalid deSelect string" + + " INVALIED_deselectParam " + "specified. It should be one of", + message); + WebServicesTestUtils.checkStringEqual("exception type", + "BadRequestException", type); + WebServicesTestUtils.checkStringEqual("exception classname", + "org.apache.hadoop.yarn.webapp.BadRequestException", classname); + } finally { + rm.stop(); + } + } + + @Test + public void testAppsQueryWithDeselect() + throws JSONException, Exception { + rm.start(); + MockNM amNodeManager = rm.registerNode("127.0.0.1:1234", 2048); + rm.submitApp(CONTAINER_MB); + amNodeManager.nodeHeartbeat(true); + WebResource r = resource(); + + MultivaluedMapImpl params = new MultivaluedMapImpl(); + params.add("deSelect", DeSelectQuery.DeSelectParam.resourceRequests.toString()); + ClientResponse response = r.path("ws").path("v1").path("cluster") + .path("apps").queryParams(params) + .accept(MediaType.APPLICATION_JSON).get(ClientResponse.class); + assertEquals(MediaType.APPLICATION_JSON_TYPE + "; " + JettyUtils.UTF_8, + response.getType().toString()); + + JSONObject json = response.getEntity(JSONObject.class); + assertEquals("incorrect number of elements", 1, json.length()); + JSONObject apps = json.getJSONObject("apps"); + assertEquals("incorrect number of elements", 1, apps.length()); + JSONArray array = apps.getJSONArray("app"); + assertEquals("incorrect number of elements", 1, array.length()); + JSONObject app = array.getJSONObject(0); + assertTrue("resource requests shouldn't exits", + !app.has("resourceRequests")); + rm.stop(); + } + + @Test public void testAppStatistics() throws JSONException, Exception { try { rm.start();