diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/DeSelectFields.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/DeSelectFields.java index 258bbfa..c991766 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/DeSelectFields.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/DeSelectFields.java @@ -91,7 +91,16 @@ public boolean contains(DeSelectType type) { * RESOURCE_REQUESTS is the first * supported type from YARN-6280. */ - RESOURCE_REQUESTS("resourceRequests"); + RESOURCE_REQUESTS("resourceRequests"), + /** + * APP_TIMEOUTS, APP_NODE_LABEL_EXPRESSION, AM_NODE_LABEL_EXPRESSION, + * RESOURCE_INFO are additionally supported parameters added in + * YARN-6871. + */ + TIMEOUTS("timeouts"), + APP_NODE_LABEL_EXPRESSION("appNodeLabelExpression"), + AM_NODE_LABEL_EXPRESSION("amNodeLabelExpression"), + RESOURCE_INFO("resourceInfo"); private final String literals; diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/AppInfo.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/AppInfo.java index f11939a..a2582b0 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/AppInfo.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/AppInfo.java @@ -120,7 +120,7 @@ protected String amNodeLabelExpression; protected ResourcesInfo resourceInfo = null; - protected AppTimeoutsInfo timeouts = new AppTimeoutsInfo(); + private AppTimeoutsInfo timeouts; public AppInfo() { } // JAXB needs this @@ -134,6 +134,7 @@ public AppInfo(ResourceManager rm, RMApp app, Boolean hasAccess, public AppInfo(ResourceManager rm, RMApp app, Boolean hasAccess, String schemePrefix, DeSelectFields deSelects) { this.schemePrefix = schemePrefix; + if (app != null) { String trackingUrl = app.getTrackingUrl(); this.state = app.createApplicationState(); @@ -142,12 +143,11 @@ public AppInfo(ResourceManager rm, RMApp app, Boolean hasAccess, || YarnApplicationState.NEW_SAVING == this.state || YarnApplicationState.SUBMITTED == this.state || YarnApplicationState.ACCEPTED == this.state; - this.trackingUI = this.trackingUrlIsNotReady ? "UNASSIGNED" : (app - .getFinishTime() == 0 ? "ApplicationMaster" : "History"); + this.trackingUI = this.trackingUrlIsNotReady ? "UNASSIGNED" + : (app.getFinishTime() == 0 ? "ApplicationMaster" : "History"); if (!trackingUrlIsNotReady) { this.trackingUrl = - WebAppUtils.getURLWithScheme(schemePrefix, - trackingUrl); + WebAppUtils.getURLWithScheme(schemePrefix, trackingUrl); this.trackingUrlPretty = this.trackingUrl; } else { this.trackingUrlPretty = "UNASSIGNED"; @@ -162,15 +162,15 @@ public AppInfo(ResourceManager rm, RMApp app, Boolean hasAccess, this.priority = 0; if (app.getApplicationPriority() != null) { - this.priority = app.getApplicationPriority() - .getPriority(); + this.priority = app.getApplicationPriority().getPriority(); } this.progress = app.getProgress() * 100; this.diagnostics = app.getDiagnostics().toString(); if (diagnostics == null || diagnostics.isEmpty()) { this.diagnostics = ""; } - if (app.getApplicationTags() != null && !app.getApplicationTags().isEmpty()) { + if (app.getApplicationTags() != null + && !app.getApplicationTags().isEmpty()) { this.applicationTags = Joiner.on(',').join(app.getApplicationTags()); } this.finalStatus = app.getFinalApplicationStatus(); @@ -178,8 +178,8 @@ public AppInfo(ResourceManager rm, RMApp app, Boolean hasAccess, if (hasAccess) { this.startedTime = app.getStartTime(); this.finishedTime = app.getFinishTime(); - this.elapsedTime = Times.elapsed(app.getStartTime(), - app.getFinishTime()); + this.elapsedTime = + Times.elapsed(app.getStartTime(), app.getFinishTime()); this.logAggregationStatus = app.getLogAggregationStatusForAppReport(); RMAppAttempt attempt = app.getCurrentAppAttempt(); if (attempt != null) { @@ -194,8 +194,8 @@ public AppInfo(ResourceManager rm, RMApp app, Boolean hasAccess, this.amRPCAddress = getAmRPCAddressFromRMAppAttempt(attempt); - ApplicationResourceUsageReport resourceReport = attempt - .getApplicationResourceUsageReport(); + ApplicationResourceUsageReport resourceReport = + attempt.getApplicationResourceUsageReport(); if (resourceReport != null) { Resource usedResources = resourceReport.getUsedResources(); Resource reservedResources = resourceReport.getReservedResources(); @@ -208,10 +208,11 @@ public AppInfo(ResourceManager rm, RMApp app, Boolean hasAccess, clusterUsagePercentage = resourceReport.getClusterUsagePercentage(); } - /* When the deSelects parameter contains "resourceRequests", - it skips returning massive ResourceRequest objects and vice versa. - Default behavior is no skipping. (YARN-6280) - */ + /* + * When the deSelects parameter contains "resourceRequests", it skips + * returning massive ResourceRequest objects and vice versa. Default + * behavior is no skipping. (YARN-6280) + */ if (!deSelects.contains(DeSelectType.RESOURCE_REQUESTS)) { List resourceRequestsRaw = rm.getRMContext() .getScheduler().getPendingResourceRequestsForAttempt( @@ -228,12 +229,9 @@ public AppInfo(ResourceManager rm, RMApp app, Boolean hasAccess, // copy preemption info fields RMAppMetrics appMetrics = app.getRMAppMetrics(); - numAMContainerPreempted = - appMetrics.getNumAMContainersPreempted(); - preemptedResourceMB = - appMetrics.getResourcePreempted().getMemorySize(); - numNonAMContainerPreempted = - appMetrics.getNumNonAMContainersPreempted(); + numAMContainerPreempted = appMetrics.getNumAMContainersPreempted(); + preemptedResourceMB = appMetrics.getResourcePreempted().getMemorySize(); + numNonAMContainerPreempted = appMetrics.getNumNonAMContainersPreempted(); preemptedResourceVCores = appMetrics.getResourcePreempted().getVirtualCores(); memorySeconds = appMetrics.getMemorySeconds(); @@ -242,48 +240,79 @@ public AppInfo(ResourceManager rm, RMApp app, Boolean hasAccess, preemptedVcoreSeconds = appMetrics.getPreemptedVcoreSeconds(); ApplicationSubmissionContext appSubmissionContext = app.getApplicationSubmissionContext(); - unmanagedApplication = - appSubmissionContext.getUnmanagedAM(); - appNodeLabelExpression = - app.getApplicationSubmissionContext().getNodeLabelExpression(); - amNodeLabelExpression = (unmanagedApplication) ? null - : app.getAMResourceRequests().get(0).getNodeLabelExpression(); + unmanagedApplication = appSubmissionContext.getUnmanagedAM(); + + /* + * When the deSelects parameter contains "appNodeLabelExpression", objects + * pertaining to the appNodeLabelExpression are not returned. By default, + * this is not skipped. (YARN-6871) + */ + if (!deSelects.contains(DeSelectType.APP_NODE_LABEL_EXPRESSION)) { + appNodeLabelExpression = + app.getApplicationSubmissionContext().getNodeLabelExpression(); + } + + /* + * When the deSelects parameter contains "amNodeLabelExpression", objects + * pertaining to the amNodeLabelExpression are not returned. By default, + * this is not skipped. (YARN-6871) + */ + if (!deSelects.contains(DeSelectType.AM_NODE_LABEL_EXPRESSION)) { + amNodeLabelExpression = (unmanagedApplication) ? + null : + app.getAMResourceRequests().get(0).getNodeLabelExpression(); + } + /* + * When the deSelects parameter contains "resourceInfo", ResourceInfo + * objects are not returned. Default behavior is no skipping. (YARN-6871) + */ // Setting partition based resource usage of application - ResourceScheduler scheduler = rm.getRMContext().getScheduler(); - if (scheduler instanceof CapacityScheduler) { - RMAppAttempt attempt = app.getCurrentAppAttempt(); - if (null != attempt) { - FiCaSchedulerApp ficaAppAttempt = ((CapacityScheduler) scheduler) - .getApplicationAttempt(attempt.getAppAttemptId()); - resourceInfo = null != ficaAppAttempt - ? new ResourcesInfo(ficaAppAttempt.getSchedulingResourceUsage()) - : null; + if (!deSelects.contains(DeSelectType.RESOURCE_INFO)) { + ResourceScheduler scheduler = rm.getRMContext().getScheduler(); + if (scheduler instanceof CapacityScheduler) { + RMAppAttempt attempt = app.getCurrentAppAttempt(); + if (null != attempt) { + FiCaSchedulerApp ficaAppAttempt = ((CapacityScheduler) scheduler) + .getApplicationAttempt(attempt.getAppAttemptId()); + resourceInfo = null != ficaAppAttempt ? + new ResourcesInfo(ficaAppAttempt.getSchedulingResourceUsage()) : + null; + } } } - Map applicationTimeouts = - app.getApplicationTimeouts(); - if (applicationTimeouts.isEmpty()) { - // If application is not set timeout, lifetime should be sent as default - // with expiryTime=UNLIMITED and remainingTime=-1 - AppTimeoutInfo timeoutInfo = new AppTimeoutInfo(); - timeoutInfo.setTimeoutType(ApplicationTimeoutType.LIFETIME); - timeouts.add(timeoutInfo); - } else { - for (Map.Entry entry : app - .getApplicationTimeouts().entrySet()) { - AppTimeoutInfo timeout = new AppTimeoutInfo(); - timeout.setTimeoutType(entry.getKey()); - long timeoutInMillis = entry.getValue().longValue(); - timeout.setExpiryTime(Times.formatISO8601(timeoutInMillis)); - if (app.isAppInCompletedStates()) { - timeout.setRemainingTime(0); - } else { - timeout.setRemainingTime(Math - .max((timeoutInMillis - System.currentTimeMillis()) / 1000, 0)); + /* + * When the deSelects parameter contains "appTimeouts", objects pertaining + * to app timeouts are not returned. By default, this is not skipped. + * (YARN-6871) + */ + if (!deSelects.contains(DeSelectType.TIMEOUTS)) { + timeouts = new AppTimeoutsInfo(); + Map applicationTimeouts = + app.getApplicationTimeouts(); + if (applicationTimeouts.isEmpty()) { + // If application is not set timeout, lifetime should be sent as + // default + // with expiryTime=UNLIMITED and remainingTime=-1 + AppTimeoutInfo timeoutInfo = new AppTimeoutInfo(); + timeoutInfo.setTimeoutType(ApplicationTimeoutType.LIFETIME); + timeouts.add(timeoutInfo); + } else { + for (Map.Entry entry : app + .getApplicationTimeouts().entrySet()) { + AppTimeoutInfo timeout = new AppTimeoutInfo(); + timeout.setTimeoutType(entry.getKey()); + long timeoutInMillis = entry.getValue().longValue(); + timeout.setExpiryTime(Times.formatISO8601(timeoutInMillis)); + if (app.isAppInCompletedStates()) { + timeout.setRemainingTime(0); + } else { + timeout.setRemainingTime(Math.max( + (timeoutInMillis - System.currentTimeMillis()) / 1000, 0)); + } + timeouts.add(timeout); } - timeouts.add(timeout); } } } @@ -396,19 +425,19 @@ public String getApplicationType() { public String getApplicationTags() { return this.applicationTags; } - + public int getRunningContainers() { return this.runningContainers; } - + public long getAllocatedMB() { return this.allocatedMB; } - + public long getAllocatedVCores() { return this.allocatedVCores; } - + public long getReservedMB() { return this.reservedMB; } @@ -428,11 +457,11 @@ public long getPreemptedVCores() { public int getNumNonAMContainersPreempted() { return numNonAMContainerPreempted; } - + public int getNumAMContainersPreempted() { return numAMContainerPreempted; } - + public long getMemorySeconds() { return memorySeconds; } @@ -448,6 +477,7 @@ public long getPreemptedMemorySeconds() { public long getPreemptedVcoreSeconds() { return preemptedVcoreSeconds; } + public List getResourceRequests() { return this.resourceRequests; } diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesApps.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesApps.java index 1cbdec3..b60e581 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesApps.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesApps.java @@ -1113,8 +1113,83 @@ public void testAppsQueryWithDeselects() JSONArray array = apps.getJSONArray("app"); assertEquals("incorrect number of elements", 1, array.length()); JSONObject app = array.getJSONObject(0); - assertTrue("resource requests shouldn't exits", + assertTrue("resource requests shouldn't exist", !app.has("resourceRequests")); + + params.clear(); + params.add("deSelects", + DeSelectFields.DeSelectType.AM_NODE_LABEL_EXPRESSION.toString()); + response = + r.path("ws").path("v1").path("cluster").path("apps").queryParams(params) + .accept(MediaType.APPLICATION_JSON).get(ClientResponse.class); + assertEquals(MediaType.APPLICATION_JSON_TYPE + "; " + JettyUtils.UTF_8, + response.getType().toString()); + + json = response.getEntity(JSONObject.class); + assertEquals("incorrect number of elements", 1, json.length()); + apps = json.getJSONObject("apps"); + assertEquals("incorrect number of elements", 1, apps.length()); + array = apps.getJSONArray("app"); + assertEquals("incorrect number of elements", 1, array.length()); + app = array.getJSONObject(0); + assertTrue("AMNodeLabelExpression shouldn't exist", + !app.has("amNodeLabelExpression")); + + params.clear(); + params.add("deSelects", DeSelectFields.DeSelectType.TIMEOUTS.toString()); + response = + r.path("ws").path("v1").path("cluster").path("apps").queryParams(params) + .accept(MediaType.APPLICATION_JSON).get(ClientResponse.class); + assertEquals(MediaType.APPLICATION_JSON_TYPE + "; " + JettyUtils.UTF_8, + response.getType().toString()); + + json = response.getEntity(JSONObject.class); + assertEquals("incorrect number of elements", 1, json.length()); + apps = json.getJSONObject("apps"); + assertEquals("incorrect number of elements", 1, apps.length()); + array = apps.getJSONArray("app"); + assertEquals("incorrect number of elements", 1, array.length()); + app = array.getJSONObject(0); + assertTrue("Timeouts shouldn't exist", !app.has("timeouts")); + rm.stop(); + + params.clear(); + params.add("deSelects", + DeSelectFields.DeSelectType.APP_NODE_LABEL_EXPRESSION.toString()); + response = + r.path("ws").path("v1").path("cluster").path("apps").queryParams(params) + .accept(MediaType.APPLICATION_JSON).get(ClientResponse.class); + assertEquals(MediaType.APPLICATION_JSON_TYPE + "; " + JettyUtils.UTF_8, + response.getType().toString()); + + json = response.getEntity(JSONObject.class); + assertEquals("incorrect number of elements", 1, json.length()); + apps = json.getJSONObject("apps"); + assertEquals("incorrect number of elements", 1, apps.length()); + array = apps.getJSONArray("app"); + assertEquals("incorrect number of elements", 1, array.length()); + app = array.getJSONObject(0); + assertTrue("AppNodeLabelExpression shouldn't exist", + !app.has("appNodeLabelExpression")); + rm.stop(); + + params.clear(); + params + .add("deSelects", DeSelectFields.DeSelectType.RESOURCE_INFO.toString()); + response = + r.path("ws").path("v1").path("cluster").path("apps").queryParams(params) + .accept(MediaType.APPLICATION_JSON).get(ClientResponse.class); + assertEquals(MediaType.APPLICATION_JSON_TYPE + "; " + JettyUtils.UTF_8, + response.getType().toString()); + + json = response.getEntity(JSONObject.class); + assertEquals("incorrect number of elements", 1, json.length()); + apps = json.getJSONObject("apps"); + assertEquals("incorrect number of elements", 1, apps.length()); + array = apps.getJSONArray("app"); + assertEquals("incorrect number of elements", 1, array.length()); + app = array.getJSONObject(0); + assertTrue("Resource info shouldn't exist", !app.has("resourceInfo")); rm.stop(); }