diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/DeSelectFields.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/DeSelectFields.java
index 258bbfa..c991766 100644
--- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/DeSelectFields.java
+++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/DeSelectFields.java
@@ -91,7 +91,16 @@ public boolean contains(DeSelectType type) {
* RESOURCE_REQUESTS is the first
* supported type from YARN-6280.
*/
- RESOURCE_REQUESTS("resourceRequests");
+ RESOURCE_REQUESTS("resourceRequests"),
+ /**
+ * APP_TIMEOUTS, APP_NODE_LABEL_EXPRESSION, AM_NODE_LABEL_EXPRESSION,
+ * RESOURCE_INFO are additionally supported parameters added in
+ * YARN-6871.
+ */
+ TIMEOUTS("timeouts"),
+ APP_NODE_LABEL_EXPRESSION("appNodeLabelExpression"),
+ AM_NODE_LABEL_EXPRESSION("amNodeLabelExpression"),
+ RESOURCE_INFO("resourceInfo");
private final String literals;
diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/AppInfo.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/AppInfo.java
index f11939a..a2582b0 100644
--- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/AppInfo.java
+++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/AppInfo.java
@@ -120,7 +120,7 @@
protected String amNodeLabelExpression;
protected ResourcesInfo resourceInfo = null;
- protected AppTimeoutsInfo timeouts = new AppTimeoutsInfo();
+ private AppTimeoutsInfo timeouts;
public AppInfo() {
} // JAXB needs this
@@ -134,6 +134,7 @@ public AppInfo(ResourceManager rm, RMApp app, Boolean hasAccess,
public AppInfo(ResourceManager rm, RMApp app, Boolean hasAccess,
String schemePrefix, DeSelectFields deSelects) {
this.schemePrefix = schemePrefix;
+
if (app != null) {
String trackingUrl = app.getTrackingUrl();
this.state = app.createApplicationState();
@@ -142,12 +143,11 @@ public AppInfo(ResourceManager rm, RMApp app, Boolean hasAccess,
|| YarnApplicationState.NEW_SAVING == this.state
|| YarnApplicationState.SUBMITTED == this.state
|| YarnApplicationState.ACCEPTED == this.state;
- this.trackingUI = this.trackingUrlIsNotReady ? "UNASSIGNED" : (app
- .getFinishTime() == 0 ? "ApplicationMaster" : "History");
+ this.trackingUI = this.trackingUrlIsNotReady ? "UNASSIGNED"
+ : (app.getFinishTime() == 0 ? "ApplicationMaster" : "History");
if (!trackingUrlIsNotReady) {
this.trackingUrl =
- WebAppUtils.getURLWithScheme(schemePrefix,
- trackingUrl);
+ WebAppUtils.getURLWithScheme(schemePrefix, trackingUrl);
this.trackingUrlPretty = this.trackingUrl;
} else {
this.trackingUrlPretty = "UNASSIGNED";
@@ -162,15 +162,15 @@ public AppInfo(ResourceManager rm, RMApp app, Boolean hasAccess,
this.priority = 0;
if (app.getApplicationPriority() != null) {
- this.priority = app.getApplicationPriority()
- .getPriority();
+ this.priority = app.getApplicationPriority().getPriority();
}
this.progress = app.getProgress() * 100;
this.diagnostics = app.getDiagnostics().toString();
if (diagnostics == null || diagnostics.isEmpty()) {
this.diagnostics = "";
}
- if (app.getApplicationTags() != null && !app.getApplicationTags().isEmpty()) {
+ if (app.getApplicationTags() != null
+ && !app.getApplicationTags().isEmpty()) {
this.applicationTags = Joiner.on(',').join(app.getApplicationTags());
}
this.finalStatus = app.getFinalApplicationStatus();
@@ -178,8 +178,8 @@ public AppInfo(ResourceManager rm, RMApp app, Boolean hasAccess,
if (hasAccess) {
this.startedTime = app.getStartTime();
this.finishedTime = app.getFinishTime();
- this.elapsedTime = Times.elapsed(app.getStartTime(),
- app.getFinishTime());
+ this.elapsedTime =
+ Times.elapsed(app.getStartTime(), app.getFinishTime());
this.logAggregationStatus = app.getLogAggregationStatusForAppReport();
RMAppAttempt attempt = app.getCurrentAppAttempt();
if (attempt != null) {
@@ -194,8 +194,8 @@ public AppInfo(ResourceManager rm, RMApp app, Boolean hasAccess,
this.amRPCAddress = getAmRPCAddressFromRMAppAttempt(attempt);
- ApplicationResourceUsageReport resourceReport = attempt
- .getApplicationResourceUsageReport();
+ ApplicationResourceUsageReport resourceReport =
+ attempt.getApplicationResourceUsageReport();
if (resourceReport != null) {
Resource usedResources = resourceReport.getUsedResources();
Resource reservedResources = resourceReport.getReservedResources();
@@ -208,10 +208,11 @@ public AppInfo(ResourceManager rm, RMApp app, Boolean hasAccess,
clusterUsagePercentage = resourceReport.getClusterUsagePercentage();
}
- /* When the deSelects parameter contains "resourceRequests",
- it skips returning massive ResourceRequest objects and vice versa.
- Default behavior is no skipping. (YARN-6280)
- */
+ /*
+ * When the deSelects parameter contains "resourceRequests", it skips
+ * returning massive ResourceRequest objects and vice versa. Default
+ * behavior is no skipping. (YARN-6280)
+ */
if (!deSelects.contains(DeSelectType.RESOURCE_REQUESTS)) {
List resourceRequestsRaw = rm.getRMContext()
.getScheduler().getPendingResourceRequestsForAttempt(
@@ -228,12 +229,9 @@ public AppInfo(ResourceManager rm, RMApp app, Boolean hasAccess,
// copy preemption info fields
RMAppMetrics appMetrics = app.getRMAppMetrics();
- numAMContainerPreempted =
- appMetrics.getNumAMContainersPreempted();
- preemptedResourceMB =
- appMetrics.getResourcePreempted().getMemorySize();
- numNonAMContainerPreempted =
- appMetrics.getNumNonAMContainersPreempted();
+ numAMContainerPreempted = appMetrics.getNumAMContainersPreempted();
+ preemptedResourceMB = appMetrics.getResourcePreempted().getMemorySize();
+ numNonAMContainerPreempted = appMetrics.getNumNonAMContainersPreempted();
preemptedResourceVCores =
appMetrics.getResourcePreempted().getVirtualCores();
memorySeconds = appMetrics.getMemorySeconds();
@@ -242,48 +240,79 @@ public AppInfo(ResourceManager rm, RMApp app, Boolean hasAccess,
preemptedVcoreSeconds = appMetrics.getPreemptedVcoreSeconds();
ApplicationSubmissionContext appSubmissionContext =
app.getApplicationSubmissionContext();
- unmanagedApplication =
- appSubmissionContext.getUnmanagedAM();
- appNodeLabelExpression =
- app.getApplicationSubmissionContext().getNodeLabelExpression();
- amNodeLabelExpression = (unmanagedApplication) ? null
- : app.getAMResourceRequests().get(0).getNodeLabelExpression();
+ unmanagedApplication = appSubmissionContext.getUnmanagedAM();
+
+ /*
+ * When the deSelects parameter contains "appNodeLabelExpression", objects
+ * pertaining to the appNodeLabelExpression are not returned. By default,
+ * this is not skipped. (YARN-6871)
+ */
+ if (!deSelects.contains(DeSelectType.APP_NODE_LABEL_EXPRESSION)) {
+ appNodeLabelExpression =
+ app.getApplicationSubmissionContext().getNodeLabelExpression();
+ }
+
+ /*
+ * When the deSelects parameter contains "amNodeLabelExpression", objects
+ * pertaining to the amNodeLabelExpression are not returned. By default,
+ * this is not skipped. (YARN-6871)
+ */
+ if (!deSelects.contains(DeSelectType.AM_NODE_LABEL_EXPRESSION)) {
+ amNodeLabelExpression = (unmanagedApplication) ?
+ null :
+ app.getAMResourceRequests().get(0).getNodeLabelExpression();
+ }
+ /*
+ * When the deSelects parameter contains "resourceInfo", ResourceInfo
+ * objects are not returned. Default behavior is no skipping. (YARN-6871)
+ */
// Setting partition based resource usage of application
- ResourceScheduler scheduler = rm.getRMContext().getScheduler();
- if (scheduler instanceof CapacityScheduler) {
- RMAppAttempt attempt = app.getCurrentAppAttempt();
- if (null != attempt) {
- FiCaSchedulerApp ficaAppAttempt = ((CapacityScheduler) scheduler)
- .getApplicationAttempt(attempt.getAppAttemptId());
- resourceInfo = null != ficaAppAttempt
- ? new ResourcesInfo(ficaAppAttempt.getSchedulingResourceUsage())
- : null;
+ if (!deSelects.contains(DeSelectType.RESOURCE_INFO)) {
+ ResourceScheduler scheduler = rm.getRMContext().getScheduler();
+ if (scheduler instanceof CapacityScheduler) {
+ RMAppAttempt attempt = app.getCurrentAppAttempt();
+ if (null != attempt) {
+ FiCaSchedulerApp ficaAppAttempt = ((CapacityScheduler) scheduler)
+ .getApplicationAttempt(attempt.getAppAttemptId());
+ resourceInfo = null != ficaAppAttempt ?
+ new ResourcesInfo(ficaAppAttempt.getSchedulingResourceUsage()) :
+ null;
+ }
}
}
- Map applicationTimeouts =
- app.getApplicationTimeouts();
- if (applicationTimeouts.isEmpty()) {
- // If application is not set timeout, lifetime should be sent as default
- // with expiryTime=UNLIMITED and remainingTime=-1
- AppTimeoutInfo timeoutInfo = new AppTimeoutInfo();
- timeoutInfo.setTimeoutType(ApplicationTimeoutType.LIFETIME);
- timeouts.add(timeoutInfo);
- } else {
- for (Map.Entry entry : app
- .getApplicationTimeouts().entrySet()) {
- AppTimeoutInfo timeout = new AppTimeoutInfo();
- timeout.setTimeoutType(entry.getKey());
- long timeoutInMillis = entry.getValue().longValue();
- timeout.setExpiryTime(Times.formatISO8601(timeoutInMillis));
- if (app.isAppInCompletedStates()) {
- timeout.setRemainingTime(0);
- } else {
- timeout.setRemainingTime(Math
- .max((timeoutInMillis - System.currentTimeMillis()) / 1000, 0));
+ /*
+ * When the deSelects parameter contains "appTimeouts", objects pertaining
+ * to app timeouts are not returned. By default, this is not skipped.
+ * (YARN-6871)
+ */
+ if (!deSelects.contains(DeSelectType.TIMEOUTS)) {
+ timeouts = new AppTimeoutsInfo();
+ Map applicationTimeouts =
+ app.getApplicationTimeouts();
+ if (applicationTimeouts.isEmpty()) {
+ // If application is not set timeout, lifetime should be sent as
+ // default
+ // with expiryTime=UNLIMITED and remainingTime=-1
+ AppTimeoutInfo timeoutInfo = new AppTimeoutInfo();
+ timeoutInfo.setTimeoutType(ApplicationTimeoutType.LIFETIME);
+ timeouts.add(timeoutInfo);
+ } else {
+ for (Map.Entry entry : app
+ .getApplicationTimeouts().entrySet()) {
+ AppTimeoutInfo timeout = new AppTimeoutInfo();
+ timeout.setTimeoutType(entry.getKey());
+ long timeoutInMillis = entry.getValue().longValue();
+ timeout.setExpiryTime(Times.formatISO8601(timeoutInMillis));
+ if (app.isAppInCompletedStates()) {
+ timeout.setRemainingTime(0);
+ } else {
+ timeout.setRemainingTime(Math.max(
+ (timeoutInMillis - System.currentTimeMillis()) / 1000, 0));
+ }
+ timeouts.add(timeout);
}
- timeouts.add(timeout);
}
}
}
@@ -396,19 +425,19 @@ public String getApplicationType() {
public String getApplicationTags() {
return this.applicationTags;
}
-
+
public int getRunningContainers() {
return this.runningContainers;
}
-
+
public long getAllocatedMB() {
return this.allocatedMB;
}
-
+
public long getAllocatedVCores() {
return this.allocatedVCores;
}
-
+
public long getReservedMB() {
return this.reservedMB;
}
@@ -428,11 +457,11 @@ public long getPreemptedVCores() {
public int getNumNonAMContainersPreempted() {
return numNonAMContainerPreempted;
}
-
+
public int getNumAMContainersPreempted() {
return numAMContainerPreempted;
}
-
+
public long getMemorySeconds() {
return memorySeconds;
}
@@ -448,6 +477,7 @@ public long getPreemptedMemorySeconds() {
public long getPreemptedVcoreSeconds() {
return preemptedVcoreSeconds;
}
+
public List getResourceRequests() {
return this.resourceRequests;
}
diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesApps.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesApps.java
index 1cbdec3..b60e581 100644
--- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesApps.java
+++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesApps.java
@@ -1113,8 +1113,83 @@ public void testAppsQueryWithDeselects()
JSONArray array = apps.getJSONArray("app");
assertEquals("incorrect number of elements", 1, array.length());
JSONObject app = array.getJSONObject(0);
- assertTrue("resource requests shouldn't exits",
+ assertTrue("resource requests shouldn't exist",
!app.has("resourceRequests"));
+
+ params.clear();
+ params.add("deSelects",
+ DeSelectFields.DeSelectType.AM_NODE_LABEL_EXPRESSION.toString());
+ response =
+ r.path("ws").path("v1").path("cluster").path("apps").queryParams(params)
+ .accept(MediaType.APPLICATION_JSON).get(ClientResponse.class);
+ assertEquals(MediaType.APPLICATION_JSON_TYPE + "; " + JettyUtils.UTF_8,
+ response.getType().toString());
+
+ json = response.getEntity(JSONObject.class);
+ assertEquals("incorrect number of elements", 1, json.length());
+ apps = json.getJSONObject("apps");
+ assertEquals("incorrect number of elements", 1, apps.length());
+ array = apps.getJSONArray("app");
+ assertEquals("incorrect number of elements", 1, array.length());
+ app = array.getJSONObject(0);
+ assertTrue("AMNodeLabelExpression shouldn't exist",
+ !app.has("amNodeLabelExpression"));
+
+ params.clear();
+ params.add("deSelects", DeSelectFields.DeSelectType.TIMEOUTS.toString());
+ response =
+ r.path("ws").path("v1").path("cluster").path("apps").queryParams(params)
+ .accept(MediaType.APPLICATION_JSON).get(ClientResponse.class);
+ assertEquals(MediaType.APPLICATION_JSON_TYPE + "; " + JettyUtils.UTF_8,
+ response.getType().toString());
+
+ json = response.getEntity(JSONObject.class);
+ assertEquals("incorrect number of elements", 1, json.length());
+ apps = json.getJSONObject("apps");
+ assertEquals("incorrect number of elements", 1, apps.length());
+ array = apps.getJSONArray("app");
+ assertEquals("incorrect number of elements", 1, array.length());
+ app = array.getJSONObject(0);
+ assertTrue("Timeouts shouldn't exist", !app.has("timeouts"));
+ rm.stop();
+
+ params.clear();
+ params.add("deSelects",
+ DeSelectFields.DeSelectType.APP_NODE_LABEL_EXPRESSION.toString());
+ response =
+ r.path("ws").path("v1").path("cluster").path("apps").queryParams(params)
+ .accept(MediaType.APPLICATION_JSON).get(ClientResponse.class);
+ assertEquals(MediaType.APPLICATION_JSON_TYPE + "; " + JettyUtils.UTF_8,
+ response.getType().toString());
+
+ json = response.getEntity(JSONObject.class);
+ assertEquals("incorrect number of elements", 1, json.length());
+ apps = json.getJSONObject("apps");
+ assertEquals("incorrect number of elements", 1, apps.length());
+ array = apps.getJSONArray("app");
+ assertEquals("incorrect number of elements", 1, array.length());
+ app = array.getJSONObject(0);
+ assertTrue("AppNodeLabelExpression shouldn't exist",
+ !app.has("appNodeLabelExpression"));
+ rm.stop();
+
+ params.clear();
+ params
+ .add("deSelects", DeSelectFields.DeSelectType.RESOURCE_INFO.toString());
+ response =
+ r.path("ws").path("v1").path("cluster").path("apps").queryParams(params)
+ .accept(MediaType.APPLICATION_JSON).get(ClientResponse.class);
+ assertEquals(MediaType.APPLICATION_JSON_TYPE + "; " + JettyUtils.UTF_8,
+ response.getType().toString());
+
+ json = response.getEntity(JSONObject.class);
+ assertEquals("incorrect number of elements", 1, json.length());
+ apps = json.getJSONObject("apps");
+ assertEquals("incorrect number of elements", 1, apps.length());
+ array = apps.getJSONArray("app");
+ assertEquals("incorrect number of elements", 1, array.length());
+ app = array.getJSONObject(0);
+ assertTrue("Resource info shouldn't exist", !app.has("resourceInfo"));
rm.stop();
}