diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/DeSelectFields.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/DeSelectFields.java index 258bbfa..0129ef2 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/DeSelectFields.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/DeSelectFields.java @@ -91,7 +91,16 @@ public boolean contains(DeSelectType type) { * RESOURCE_REQUESTS is the first * supported type from YARN-6280. */ - RESOURCE_REQUESTS("resourceRequests"); + RESOURCE_REQUESTS("resourceRequests"), + /** + * APP_TIMEOUTS, APP_NODE_LABEL_EXPRESSION, AM_NODE_LABEL_EXPRESSION, + * RESOURCE_INFO are additionally supported parameters added in + * YARN-6871. + */ + APP_TIMEOUTS("appTimeouts"), + APP_NODE_LABEL_EXPRESSION("appNodeLabelExpression"), + AM_NODE_LABEL_EXPRESSION("amNodeLabelExpression"), + RESOURCE_INFO("resourceInfo"); private final String literals; diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/AppInfo.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/AppInfo.java index f11939a..e37b4c7 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/AppInfo.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/AppInfo.java @@ -112,7 +112,7 @@ // list of resource requests @XmlElement(name = "resourceRequests") private List resourceRequests = - new ArrayList(); + new ArrayList(); protected LogAggregationStatus logAggregationStatus; protected boolean unmanagedApplication; @@ -120,34 +120,35 @@ protected String amNodeLabelExpression; protected ResourcesInfo resourceInfo = null; - protected AppTimeoutsInfo timeouts = new AppTimeoutsInfo(); + protected AppTimeoutsInfo timeouts; public AppInfo() { } // JAXB needs this public AppInfo(ResourceManager rm, RMApp app, Boolean hasAccess, - String schemePrefix) { + String schemePrefix) { this(rm, app, hasAccess, schemePrefix, new DeSelectFields()); } @SuppressWarnings({ "rawtypes", "unchecked" }) public AppInfo(ResourceManager rm, RMApp app, Boolean hasAccess, - String schemePrefix, DeSelectFields deSelects) { + String schemePrefix, DeSelectFields deSelects) { this.schemePrefix = schemePrefix; + if (app != null) { String trackingUrl = app.getTrackingUrl(); this.state = app.createApplicationState(); this.trackingUrlIsNotReady = trackingUrl == null || trackingUrl.isEmpty() - || YarnApplicationState.NEW == this.state - || YarnApplicationState.NEW_SAVING == this.state - || YarnApplicationState.SUBMITTED == this.state - || YarnApplicationState.ACCEPTED == this.state; + || YarnApplicationState.NEW == this.state + || YarnApplicationState.NEW_SAVING == this.state + || YarnApplicationState.SUBMITTED == this.state + || YarnApplicationState.ACCEPTED == this.state; this.trackingUI = this.trackingUrlIsNotReady ? "UNASSIGNED" : (app - .getFinishTime() == 0 ? "ApplicationMaster" : "History"); + .getFinishTime() == 0 ? "ApplicationMaster" : "History"); if (!trackingUrlIsNotReady) { this.trackingUrl = - WebAppUtils.getURLWithScheme(schemePrefix, - trackingUrl); + WebAppUtils.getURLWithScheme(schemePrefix, + trackingUrl); this.trackingUrlPretty = this.trackingUrl; } else { this.trackingUrlPretty = "UNASSIGNED"; @@ -163,7 +164,7 @@ public AppInfo(ResourceManager rm, RMApp app, Boolean hasAccess, if (app.getApplicationPriority() != null) { this.priority = app.getApplicationPriority() - .getPriority(); + .getPriority(); } this.progress = app.getProgress() * 100; this.diagnostics = app.getDiagnostics().toString(); @@ -179,7 +180,7 @@ public AppInfo(ResourceManager rm, RMApp app, Boolean hasAccess, this.startedTime = app.getStartTime(); this.finishedTime = app.getFinishTime(); this.elapsedTime = Times.elapsed(app.getStartTime(), - app.getFinishTime()); + app.getFinishTime()); this.logAggregationStatus = app.getLogAggregationStatusForAppReport(); RMAppAttempt attempt = app.getCurrentAppAttempt(); if (attempt != null) { @@ -187,15 +188,15 @@ public AppInfo(ResourceManager rm, RMApp app, Boolean hasAccess, if (masterContainer != null) { this.amContainerLogsExist = true; this.amContainerLogs = WebAppUtils.getRunningLogURL( - schemePrefix + masterContainer.getNodeHttpAddress(), - masterContainer.getId().toString(), app.getUser()); + schemePrefix + masterContainer.getNodeHttpAddress(), + masterContainer.getId().toString(), app.getUser()); this.amHostHttpAddress = masterContainer.getNodeHttpAddress(); } this.amRPCAddress = getAmRPCAddressFromRMAppAttempt(attempt); ApplicationResourceUsageReport resourceReport = attempt - .getApplicationResourceUsageReport(); + .getApplicationResourceUsageReport(); if (resourceReport != null) { Resource usedResources = resourceReport.getUsedResources(); Resource reservedResources = resourceReport.getReservedResources(); @@ -214,8 +215,8 @@ public AppInfo(ResourceManager rm, RMApp app, Boolean hasAccess, */ if (!deSelects.contains(DeSelectType.RESOURCE_REQUESTS)) { List resourceRequestsRaw = rm.getRMContext() - .getScheduler().getPendingResourceRequestsForAttempt( - attempt.getAppAttemptId()); + .getScheduler().getPendingResourceRequestsForAttempt( + attempt.getAppAttemptId()); if (resourceRequestsRaw != null) { for (ResourceRequest req : resourceRequestsRaw) { @@ -229,61 +230,86 @@ public AppInfo(ResourceManager rm, RMApp app, Boolean hasAccess, // copy preemption info fields RMAppMetrics appMetrics = app.getRMAppMetrics(); numAMContainerPreempted = - appMetrics.getNumAMContainersPreempted(); + appMetrics.getNumAMContainersPreempted(); preemptedResourceMB = - appMetrics.getResourcePreempted().getMemorySize(); + appMetrics.getResourcePreempted().getMemorySize(); numNonAMContainerPreempted = - appMetrics.getNumNonAMContainersPreempted(); + appMetrics.getNumNonAMContainersPreempted(); preemptedResourceVCores = - appMetrics.getResourcePreempted().getVirtualCores(); + appMetrics.getResourcePreempted().getVirtualCores(); memorySeconds = appMetrics.getMemorySeconds(); vcoreSeconds = appMetrics.getVcoreSeconds(); preemptedMemorySeconds = appMetrics.getPreemptedMemorySeconds(); preemptedVcoreSeconds = appMetrics.getPreemptedVcoreSeconds(); ApplicationSubmissionContext appSubmissionContext = - app.getApplicationSubmissionContext(); + app.getApplicationSubmissionContext(); unmanagedApplication = - appSubmissionContext.getUnmanagedAM(); - appNodeLabelExpression = - app.getApplicationSubmissionContext().getNodeLabelExpression(); - amNodeLabelExpression = (unmanagedApplication) ? null - : app.getAMResourceRequests().get(0).getNodeLabelExpression(); + appSubmissionContext.getUnmanagedAM(); + + /*When the deSelects parameter contains "appNodeLabelExpression", + objects pertaining to the appNodeLabelExpression are not returned. + By default, this is not skipped. (YARN-6871) + */ + if(!deSelects.contains(DeSelectType.APP_NODE_LABEL_EXPRESSION)) { + appNodeLabelExpression = + app.getApplicationSubmissionContext().getNodeLabelExpression(); + } + + /*When the deSelects parameter contains "amNodeLabelExpression", + objects pertaining to the amNodeLabelExpression are not returned. + By default, this is not skipped. (YARN-6871) + */ + if(!deSelects.contains(DeSelectType.AM_NODE_LABEL_EXPRESSION)) { + amNodeLabelExpression = (unmanagedApplication) ? null + : app.getAMResourceRequests().get(0).getNodeLabelExpression(); + } + /* When the deSelects parameter contains "resourceInfo", + ResourceInfo objects are not returned. + Default behavior is no skipping. (YARN-6871) + */ // Setting partition based resource usage of application - ResourceScheduler scheduler = rm.getRMContext().getScheduler(); - if (scheduler instanceof CapacityScheduler) { - RMAppAttempt attempt = app.getCurrentAppAttempt(); - if (null != attempt) { - FiCaSchedulerApp ficaAppAttempt = ((CapacityScheduler) scheduler) - .getApplicationAttempt(attempt.getAppAttemptId()); - resourceInfo = null != ficaAppAttempt - ? new ResourcesInfo(ficaAppAttempt.getSchedulingResourceUsage()) - : null; + if(!deSelects.contains(DeSelectType.RESOURCE_INFO)) { + ResourceScheduler scheduler = rm.getRMContext().getScheduler(); + if (scheduler instanceof CapacityScheduler) { + RMAppAttempt attempt = app.getCurrentAppAttempt(); + if (null != attempt) { + FiCaSchedulerApp ficaAppAttempt = ((CapacityScheduler) scheduler) + .getApplicationAttempt(attempt.getAppAttemptId()); + resourceInfo = null != ficaAppAttempt + ? new ResourcesInfo(ficaAppAttempt.getSchedulingResourceUsage()) + : null; + } } } - Map applicationTimeouts = - app.getApplicationTimeouts(); - if (applicationTimeouts.isEmpty()) { - // If application is not set timeout, lifetime should be sent as default - // with expiryTime=UNLIMITED and remainingTime=-1 - AppTimeoutInfo timeoutInfo = new AppTimeoutInfo(); - timeoutInfo.setTimeoutType(ApplicationTimeoutType.LIFETIME); - timeouts.add(timeoutInfo); - } else { - for (Map.Entry entry : app - .getApplicationTimeouts().entrySet()) { - AppTimeoutInfo timeout = new AppTimeoutInfo(); - timeout.setTimeoutType(entry.getKey()); - long timeoutInMillis = entry.getValue().longValue(); - timeout.setExpiryTime(Times.formatISO8601(timeoutInMillis)); - if (app.isAppInCompletedStates()) { - timeout.setRemainingTime(0); - } else { - timeout.setRemainingTime(Math - .max((timeoutInMillis - System.currentTimeMillis()) / 1000, 0)); + /*When the deSelects parameter contains "appTimeouts", + objects pertaining to app timeouts are not returned. + By default, this is not skipped. (YARN-6871) + */ + if(!deSelects.contains(DeSelectType.APP_TIMEOUTS)) { + timeouts = new AppTimeoutsInfo(); + Map applicationTimeouts = app.getApplicationTimeouts(); + if (applicationTimeouts.isEmpty()) { + // If application is not set timeout, lifetime should be sent as default + // with expiryTime=UNLIMITED and remainingTime=-1 + AppTimeoutInfo timeoutInfo = new AppTimeoutInfo(); + timeoutInfo.setTimeoutType(ApplicationTimeoutType.LIFETIME); + timeouts.add(timeoutInfo); + } else { + for (Map.Entry entry : app.getApplicationTimeouts().entrySet()) { + AppTimeoutInfo timeout = new AppTimeoutInfo(); + timeout.setTimeoutType(entry.getKey()); + long timeoutInMillis = entry.getValue().longValue(); + timeout.setExpiryTime(Times.formatISO8601(timeoutInMillis)); + if (app.isAppInCompletedStates()) { + timeout.setRemainingTime(0); + } else { + timeout.setRemainingTime(Math.max( + (timeoutInMillis - System.currentTimeMillis()) / 1000, 0)); + } + timeouts.add(timeout); } - timeouts.add(timeout); } } } @@ -396,19 +422,19 @@ public String getApplicationType() { public String getApplicationTags() { return this.applicationTags; } - + public int getRunningContainers() { return this.runningContainers; } - + public long getAllocatedMB() { return this.allocatedMB; } - + public long getAllocatedVCores() { return this.allocatedVCores; } - + public long getReservedMB() { return this.reservedMB; } @@ -428,11 +454,11 @@ public long getPreemptedVCores() { public int getNumNonAMContainersPreempted() { return numNonAMContainerPreempted; } - + public int getNumAMContainersPreempted() { return numAMContainerPreempted; } - + public long getMemorySeconds() { return memorySeconds; } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesApps.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesApps.java index 1cbdec3..5de64d8 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesApps.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesApps.java @@ -1113,8 +1113,46 @@ public void testAppsQueryWithDeselects() JSONArray array = apps.getJSONArray("app"); assertEquals("incorrect number of elements", 1, array.length()); JSONObject app = array.getJSONObject(0); - assertTrue("resource requests shouldn't exits", + assertTrue("resource requests shouldn't exist", !app.has("resourceRequests")); + + params.clear(); + params.add("deSelects", + DeSelectFields.DeSelectType.AM_NODE_LABEL_EXPRESSION.toString()); + response = r.path("ws").path("v1").path("cluster") + .path("apps").queryParams(params) + .accept(MediaType.APPLICATION_JSON).get(ClientResponse.class); + assertEquals(MediaType.APPLICATION_JSON_TYPE + "; " + JettyUtils.UTF_8, + response.getType().toString()); + + json = response.getEntity(JSONObject.class); + assertEquals("incorrect number of elements", 1, json.length()); + apps = json.getJSONObject("apps"); + assertEquals("incorrect number of elements", 1, apps.length()); + array = apps.getJSONArray("app"); + assertEquals("incorrect number of elements", 1, array.length()); + app = array.getJSONObject(0); + assertTrue("AMNodeLabelExpression shouldn't exist", + !app.has("amNodeLabelExpression")); + + params.clear(); + params.add("deSelects", + DeSelectFields.DeSelectType.APP_TIMEOUTS.toString()); + response = r.path("ws").path("v1").path("cluster") + .path("apps").queryParams(params) + .accept(MediaType.APPLICATION_JSON).get(ClientResponse.class); + assertEquals(MediaType.APPLICATION_JSON_TYPE + "; " + JettyUtils.UTF_8, + response.getType().toString()); + + json = response.getEntity(JSONObject.class); + assertEquals("incorrect number of elements", 1, json.length()); + apps = json.getJSONObject("apps"); + assertEquals("incorrect number of elements", 1, apps.length()); + array = apps.getJSONArray("app"); + assertEquals("incorrect number of elements", 1, array.length()); + app = array.getJSONObject(0); + assertTrue("App Timeouts shouldn't exist", + !app.has("appTimeouts")); rm.stop(); }