diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/DeSelectQuery.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/DeSelectQuery.java
new file mode 100644
index 0000000..a3b5b59
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/DeSelectQuery.java
@@ -0,0 +1,99 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.yarn.server.resourcemanager.webapp;
+
+import java.util.Arrays;
+import java.util.HashSet;
+import java.util.Set;
+
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.yarn.webapp.BadRequestException;
+
+/**
+ * DeSelectQuery make the /apps api more flexible.
+ * It can be used to strip off more fields if there's such use case in future.
+ * The process should follow two steps:
+ *
1. add a literal equivalent parameter XYZ in DeSelectParam
+ *
2. add a public boolean deSelectXYZ() in DeSelectQuery
+ */
+public class DeSelectQuery {
+ private static final Log LOG =
+ LogFactory.getLog(DeSelectQuery.class.getName());
+
+ private final Set deSelectParams;
+
+ public DeSelectQuery() {
+ this.deSelectParams = new HashSet();
+ }
+
+ /**
+ * Initial DeSelect Query with unselected fields
+ * @param unselectedFields a set of unselected field.
+ */
+ public void initQuery(Set unselectedFields) {
+ if (unselectedFields == null) {
+ return;
+ }
+ for (String field : unselectedFields) {
+ if (!field.trim().isEmpty()) {
+ String[] paramArray = field.split(",");
+ for (String param : paramArray) {
+ if (param != null && !param.trim().isEmpty()) {
+ try {
+ DeSelectParam.valueOf(param.trim());
+ } catch (RuntimeException e) {
+ LOG.warn("Invalid deSelect string " + param.trim());
+ DeSelectParam[] typeArray =
+ DeSelectParam.values();
+ String allDeSelectTypes = Arrays.toString(typeArray);
+ throw new BadRequestException("Invalid deSelect string "
+ + param.trim() + " specified. It should be one of "
+ + allDeSelectTypes);
+ }
+ this.deSelectParams.add(DeSelectParam.valueOf(param.trim()));
+ }
+ }
+ }
+ }
+ }
+
+ /**
+ * Determine resourceRequests deselected or not in query
+ * @return true if resourceRequests is deselected
+ */
+ public boolean deSelectResourceRequests() {
+ return deSelectParams.contains(DeSelectParam.resourceRequests);
+ }
+
+ /**
+ * Unselected parameter in query DeSelectQuery.
+ * Could be boost in future.
+ */
+ public enum DeSelectParam {
+
+ /**
+ * resourceRequests is the first supported parameter
+ * from YARN-6280.
+ *
It must be literal same with query parameter.
+ *
Usage: deselect="resourceRequests,.."
+ */
+ resourceRequests,
+
+ }
+}
\ No newline at end of file
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMWebServices.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMWebServices.java
index bd0602b..cd6aeae 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMWebServices.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMWebServices.java
@@ -432,7 +432,8 @@ public AppsInfo getApps(@Context HttpServletRequest hsr,
@QueryParam("finishedTimeBegin") String finishBegin,
@QueryParam("finishedTimeEnd") String finishEnd,
@QueryParam("applicationTypes") Set applicationTypes,
- @QueryParam("applicationTags") Set applicationTags) {
+ @QueryParam("applicationTags") Set applicationTags,
+ @QueryParam("deSelect") Set unselectedFields) {
boolean checkCount = false;
boolean checkStart = false;
boolean checkEnd = false;
@@ -588,8 +589,11 @@ public AppsInfo getApps(@Context HttpServletRequest hsr,
}
}
- AppInfo app = new AppInfo(rm, rmapp,
- hasAccess(rmapp, hsr), WebAppUtils.getHttpSchemePrefix(conf));
+ DeSelectQuery deSelectQuery = new DeSelectQuery();
+ deSelectQuery.initQuery(unselectedFields);
+
+ AppInfo app = new AppInfo(rm, rmapp, hasAccess(rmapp, hsr),
+ WebAppUtils.getHttpSchemePrefix(conf), deSelectQuery);
allApps.add(app);
}
return allApps;
@@ -808,14 +812,20 @@ private static void countApp(
@Produces({ MediaType.APPLICATION_JSON + "; " + JettyUtils.UTF_8,
MediaType.APPLICATION_XML + "; " + JettyUtils.UTF_8 })
public AppInfo getApp(@Context HttpServletRequest hsr,
- @PathParam("appid") String appId) {
+ @PathParam("appid") String appId,
+ @QueryParam("deSelect") Set unselectedFields) {
init();
ApplicationId id = WebAppUtils.parseApplicationId(recordFactory, appId);
RMApp app = rm.getRMContext().getRMApps().get(id);
if (app == null) {
throw new NotFoundException("app with id: " + appId + " not found");
}
- return new AppInfo(rm, app, hasAccess(app, hsr), hsr.getScheme() + "://");
+
+ DeSelectQuery deSelectQuery = new DeSelectQuery();
+ deSelectQuery.initQuery(unselectedFields);
+
+ return new AppInfo(rm, app, hasAccess(app, hsr),
+ hsr.getScheme() + "://", deSelectQuery);
}
@GET
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/AppInfo.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/AppInfo.java
index 10e627a..8b8d3eb 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/AppInfo.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/AppInfo.java
@@ -44,6 +44,7 @@
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.ResourceScheduler;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacityScheduler;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.common.fica.FiCaSchedulerApp;
+import org.apache.hadoop.yarn.server.resourcemanager.webapp.DeSelectQuery;
import org.apache.hadoop.yarn.util.Times;
import org.apache.hadoop.yarn.webapp.util.WebAppUtils;
@@ -121,9 +122,14 @@
public AppInfo() {
} // JAXB needs this
- @SuppressWarnings({ "rawtypes", "unchecked" })
public AppInfo(ResourceManager rm, RMApp app, Boolean hasAccess,
String schemePrefix) {
+ this(rm, app, hasAccess, schemePrefix, new DeSelectQuery());
+ }
+
+ @SuppressWarnings({ "rawtypes", "unchecked" })
+ public AppInfo(ResourceManager rm, RMApp app, Boolean hasAccess,
+ String schemePrefix, DeSelectQuery deSelectQuery) {
this.schemePrefix = schemePrefix;
if (app != null) {
String trackingUrl = app.getTrackingUrl();
@@ -196,13 +202,19 @@ public AppInfo(ResourceManager rm, RMApp app, Boolean hasAccess,
clusterUsagePercentage = resourceReport.getClusterUsagePercentage();
}
- List resourceRequestsRaw = rm.getRMContext()
- .getScheduler()
- .getPendingResourceRequestsForAttempt(attempt.getAppAttemptId());
-
- if (resourceRequestsRaw != null) {
- for (ResourceRequest req : resourceRequestsRaw) {
- resourceRequests.add(new ResourceRequestInfo(req));
+ /* When the deSelect parameter contains "resourceRequests",
+ it skips returning massive ResourceRequest objects.
+ Default behavior is no skipping. (YARN-6280)
+ */
+ if (!deSelectQuery.deSelectResourceRequests()) {
+ List resourceRequestsRaw = rm.getRMContext()
+ .getScheduler()
+ .getPendingResourceRequestsForAttempt(attempt.getAppAttemptId());
+
+ if (resourceRequestsRaw != null) {
+ for (ResourceRequest req : resourceRequestsRaw) {
+ resourceRequests.add(new ResourceRequestInfo(req));
+ }
}
}
}
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServices.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServices.java
index 6af2110..4f7ab54 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServices.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServices.java
@@ -665,12 +665,12 @@ public void testAppsRace() throws Exception {
// verify we don't get any apps when querying
HttpServletRequest mockHsr = mock(HttpServletRequest.class);
AppsInfo appsInfo = webSvc.getApps(mockHsr, null, emptySet, null,
- null, null, null, null, null, null, null, emptySet, emptySet);
+ null, null, null, null, null, null, null, emptySet, emptySet, null);
assertTrue(appsInfo.getApps().isEmpty());
// verify we don't get an NPE when specifying a final status query
appsInfo = webSvc.getApps(mockHsr, null, emptySet, "FAILED",
- null, null, null, null, null, null, null, emptySet, emptySet);
+ null, null, null, null, null, null, null, emptySet, emptySet, null);
assertTrue(appsInfo.getApps().isEmpty());
}
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesApps.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesApps.java
index aab9bee..60cefa5 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesApps.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesApps.java
@@ -1055,6 +1055,69 @@ public void testAppsQueryAppTypes() throws JSONException, Exception {
}
@Test
+ public void testAppsQueryWithInvaildDeselect()
+ throws JSONException, Exception {
+ try {
+ rm.start();
+ MockNM amNodeManager = rm.registerNode("127.0.0.1:1234", 2048);
+ rm.submitApp(CONTAINER_MB);
+ amNodeManager.nodeHeartbeat(true);
+ WebResource r = resource();
+ ClientResponse response = r.path("ws").path("v1").path("cluster")
+ .path("apps").queryParam("deSelect", "INVALIED_deselectParam")
+ .accept(MediaType.APPLICATION_JSON).get(ClientResponse.class);
+ assertResponseStatusCode(Status.BAD_REQUEST, response.getStatusInfo());
+ assertEquals(MediaType.APPLICATION_JSON_TYPE + "; " + JettyUtils.UTF_8,
+ response.getType().toString());
+ JSONObject msg = response.getEntity(JSONObject.class);
+ JSONObject exception = msg.getJSONObject("RemoteException");
+ assertEquals("incorrect number of elements", 3, exception.length());
+ String message = exception.getString("message");
+ String type = exception.getString("exception");
+ String classname = exception.getString("javaClassName");
+ WebServicesTestUtils.checkStringContains("exception message",
+ "java.lang.Exception: Invalid deSelect string"
+ + " INVALIED_deselectParam " + "specified. It should be one of",
+ message);
+ WebServicesTestUtils.checkStringEqual("exception type",
+ "BadRequestException", type);
+ WebServicesTestUtils.checkStringEqual("exception classname",
+ "org.apache.hadoop.yarn.webapp.BadRequestException", classname);
+ } finally {
+ rm.stop();
+ }
+ }
+
+ @Test
+ public void testAppsQueryWithDeselect()
+ throws JSONException, Exception {
+ rm.start();
+ MockNM amNodeManager = rm.registerNode("127.0.0.1:1234", 2048);
+ rm.submitApp(CONTAINER_MB);
+ amNodeManager.nodeHeartbeat(true);
+ WebResource r = resource();
+
+ MultivaluedMapImpl params = new MultivaluedMapImpl();
+ params.add("deSelect", DeSelectQuery.DeSelectParam.resourceRequests.toString());
+ ClientResponse response = r.path("ws").path("v1").path("cluster")
+ .path("apps").queryParams(params)
+ .accept(MediaType.APPLICATION_JSON).get(ClientResponse.class);
+ assertEquals(MediaType.APPLICATION_JSON_TYPE + "; " + JettyUtils.UTF_8,
+ response.getType().toString());
+
+ JSONObject json = response.getEntity(JSONObject.class);
+ assertEquals("incorrect number of elements", 1, json.length());
+ JSONObject apps = json.getJSONObject("apps");
+ assertEquals("incorrect number of elements", 1, apps.length());
+ JSONArray array = apps.getJSONArray("app");
+ assertEquals("incorrect number of elements", 1, array.length());
+ JSONObject app = array.getJSONObject(0);
+ assertTrue("resource requests shouldn't exits",
+ !app.has("resourceRequests"));
+ rm.stop();
+ }
+
+ @Test
public void testAppStatistics() throws JSONException, Exception {
try {
rm.start();