diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/activities/ActivitiesManager.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/activities/ActivitiesManager.java index b8ef2637474..766523f2386 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/activities/ActivitiesManager.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/activities/ActivitiesManager.java @@ -121,7 +121,7 @@ private void setupConfForCleanup(Configuration conf) { public AppActivitiesInfo getAppActivitiesInfo(ApplicationId applicationId, Set requestPriorities, Set allocationRequestIds, - RMWSConsts.ActivitiesGroupBy groupBy) { + RMWSConsts.ActivitiesGroupBy groupBy, int limit) { RMApp app = rmContext.getRMApps().get(applicationId); if (app != null && app.getFinalApplicationStatus() == FinalApplicationStatus.UNDEFINED) { @@ -140,6 +140,10 @@ public AppActivitiesInfo getAppActivitiesInfo(ApplicationId applicationId, allocations = new ArrayList(curAllocations); } } + if (limit > 0 && limit < allocations.size()) { + allocations = + allocations.subList(allocations.size() - limit, allocations.size()); + } return new AppActivitiesInfo(allocations, applicationId, groupBy); } else { return new AppActivitiesInfo( diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMWebServiceProtocol.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMWebServiceProtocol.java index 7b49ed4ac22..efc0225d6c6 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMWebServiceProtocol.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMWebServiceProtocol.java @@ -227,11 +227,12 @@ ActivitiesInfo getActivities(HttpServletRequest hsr, String nodeId, * the activities. It is a QueryParam. * @param groupBy the groupBy type by which the activities should be * aggregated. It is a QueryParam. + * @param limit set a limit of the result. It is a QueryParam. * @return all the activities about a specific app for a specific time */ AppActivitiesInfo getAppActivities(HttpServletRequest hsr, String appId, String time, Set requestPriorities, - Set allocationRequestIds, String groupBy); + Set allocationRequestIds, String groupBy, String limit); /** * This method retrieves all the statistics for a specific app, and it is diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMWebServices.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMWebServices.java index 9b36995eb26..0523d1f8da5 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMWebServices.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMWebServices.java @@ -719,7 +719,8 @@ public AppActivitiesInfo getAppActivities(@Context HttpServletRequest hsr, @QueryParam(RMWSConsts.REQUEST_PRIORITIES) Set requestPriorities, @QueryParam(RMWSConsts.ALLOCATION_REQUEST_IDS) Set allocationRequestIds, - @QueryParam(RMWSConsts.GROUP_BY) String groupBy) { + @QueryParam(RMWSConsts.GROUP_BY) String groupBy, + @QueryParam(RMWSConsts.LIMIT) String limit) { initForReadableEndpoints(); YarnScheduler scheduler = rm.getRMContext().getScheduler(); @@ -746,6 +747,19 @@ public AppActivitiesInfo getAppActivities(@Context HttpServletRequest hsr, return new AppActivitiesInfo(e.getMessage(), appId); } + int limitNum = -1; + if (limit != null) { + try { + limitNum = Integer.parseInt(limit); + if (limitNum <= 0) { + return new AppActivitiesInfo( + "limit must be greater than 0!", appId); + } + } catch (NumberFormatException e) { + return new AppActivitiesInfo("limit must be integer!", appId); + } + } + double maxTime = 3.0; if (time != null) { @@ -760,9 +774,9 @@ public AppActivitiesInfo getAppActivities(@Context HttpServletRequest hsr, try { applicationId = ApplicationId.fromString(appId); activitiesManager.turnOnAppActivitiesRecording(applicationId, maxTime); - AppActivitiesInfo appActivitiesInfo = - activitiesManager.getAppActivitiesInfo(applicationId, - requestPriorities, allocationRequestIds, activitiesGroupBy); + AppActivitiesInfo appActivitiesInfo = activitiesManager + .getAppActivitiesInfo(applicationId, requestPriorities, + allocationRequestIds, activitiesGroupBy, limitNum); return appActivitiesInfo; } catch (Exception e) { diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/activities/TestActivitiesManager.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/activities/TestActivitiesManager.java index 495c7e248b0..12e52c1b5ce 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/activities/TestActivitiesManager.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/activities/TestActivitiesManager.java @@ -286,14 +286,14 @@ public void testAppActivitiesTTL() throws Exception { ActivityDiagnosticConstant.SKIPPED_ALL_PRIORITIES); } AppActivitiesInfo appActivitiesInfo = newActivitiesManager - .getAppActivitiesInfo(app.getApplicationId(), null, null, null); + .getAppActivitiesInfo(app.getApplicationId(), null, null, null, -1); Assert.assertEquals(numActivities, appActivitiesInfo.getAllocations().size()); // sleep until all app activities expired Thread.sleep(cleanupIntervalMs + appActivitiesTTL); // there should be no remaining app activities appActivitiesInfo = newActivitiesManager - .getAppActivitiesInfo(app.getApplicationId(), null, null, null); + .getAppActivitiesInfo(app.getApplicationId(), null, null, null, -1); Assert.assertEquals(0, appActivitiesInfo.getAllocations().size()); } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesSchedulerActivities.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesSchedulerActivities.java index 7650f7acf1f..c96e170f38a 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesSchedulerActivities.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesSchedulerActivities.java @@ -1064,4 +1064,74 @@ public void testAppFilterByRequestPrioritiesAndAllocationRequestIds() rm.stop(); } } + + @Test(timeout = 30000) + public void testAppLimit() throws Exception { + rm.start(); + CapacityScheduler cs = (CapacityScheduler) rm.getResourceScheduler(); + + MockNM nm1 = rm.registerNode("127.0.0.1:1234", 4 * 1024); + MockNM nm2 = rm.registerNode("127.0.0.2:1234", 8 * 1024); + + try { + RMApp app1 = rm.submitApp(512, "app1", "user1", null, "b1"); + MockAM am1 = MockRM.launchAndRegisterAM(app1, rm, nm1); + + WebResource r = resource().path(RMWSConsts.RM_WEB_SERVICE_PATH) + .path(RMWSConsts.SCHEDULER_APP_ACTIVITIES); + MultivaluedMapImpl params = new MultivaluedMapImpl(); + params.add("appId", app1.getApplicationId().toString()); + JSONObject json = ActivitiesTestUtils.requestWebResource(r, params); + assertEquals("waiting for display", + json.getString("diagnostic")); + + // am1 asks for 1 * 5GB container + am1.allocate("*", 5120, 1, new ArrayList<>()); + // trigger scheduling triple, there will be 3 app activities in cache + cs.handle(new NodeUpdateSchedulerEvent( + rm.getRMContext().getRMNodes().get(nm1.getNodeId()))); + cs.handle(new NodeUpdateSchedulerEvent( + rm.getRMContext().getRMNodes().get(nm1.getNodeId()))); + cs.handle(new NodeUpdateSchedulerEvent( + rm.getRMContext().getRMNodes().get(nm1.getNodeId()))); + + // query all app activities without limit + json = ActivitiesTestUtils.requestWebResource(r, params); + verifyNumberOfAllocations(json, 3); + + // query all app activities with limit > 3 + params.putSingle(RMWSConsts.LIMIT, "10"); + json = ActivitiesTestUtils.requestWebResource(r, params); + verifyNumberOfAllocations(json, 3); + + // query app activities with limit = 2 + params.putSingle(RMWSConsts.LIMIT, "2"); + json = ActivitiesTestUtils.requestWebResource(r, params); + verifyNumberOfAllocations(json, 2); + + // query app activities with limit = 1 + params.putSingle(RMWSConsts.LIMIT, "1"); + json = ActivitiesTestUtils.requestWebResource(r, params); + verifyNumberOfAllocations(json, 1); + + // query all app activities with invalid limit + params.putSingle(RMWSConsts.LIMIT, "STRING"); + json = ActivitiesTestUtils.requestWebResource(r, params); + assertEquals("limit must be integer!", json.getString("diagnostic")); + + // query all app activities with limit = 0 + params.putSingle(RMWSConsts.LIMIT, "0"); + json = ActivitiesTestUtils.requestWebResource(r, params); + assertEquals("limit must be greater than 0!", + json.getString("diagnostic")); + + // query all app activities with limit < 0 + params.putSingle(RMWSConsts.LIMIT, "-3"); + json = ActivitiesTestUtils.requestWebResource(r, params); + assertEquals("limit must be greater than 0!", + json.getString("diagnostic")); + } finally { + rm.stop(); + } + } } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/main/java/org/apache/hadoop/yarn/server/router/webapp/DefaultRequestInterceptorREST.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/main/java/org/apache/hadoop/yarn/server/router/webapp/DefaultRequestInterceptorREST.java index f5f549a0594..003cf2a35c5 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/main/java/org/apache/hadoop/yarn/server/router/webapp/DefaultRequestInterceptorREST.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/main/java/org/apache/hadoop/yarn/server/router/webapp/DefaultRequestInterceptorREST.java @@ -192,7 +192,7 @@ public ActivitiesInfo getActivities(HttpServletRequest hsr, String nodeId, @Override public AppActivitiesInfo getAppActivities(HttpServletRequest hsr, String appId, String time, Set requestPriorities, - Set allocationRequestIds, String groupBy) { + Set allocationRequestIds, String groupBy, String limit) { // time and appId are specified inside hsr return RouterWebServiceUtil.genericForward(webAppAddress, hsr, AppActivitiesInfo.class, HTTPMethods.GET, diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/main/java/org/apache/hadoop/yarn/server/router/webapp/FederationInterceptorREST.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/main/java/org/apache/hadoop/yarn/server/router/webapp/FederationInterceptorREST.java index ec4cb8b21e0..18906baaed7 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/main/java/org/apache/hadoop/yarn/server/router/webapp/FederationInterceptorREST.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/main/java/org/apache/hadoop/yarn/server/router/webapp/FederationInterceptorREST.java @@ -1146,7 +1146,7 @@ public ActivitiesInfo getActivities(HttpServletRequest hsr, String nodeId, @Override public AppActivitiesInfo getAppActivities(HttpServletRequest hsr, String appId, String time, Set requestPriorities, - Set allocationRequestIds, String groupBy) { + Set allocationRequestIds, String groupBy, String limit) { throw new NotImplementedException("Code is not implemented"); } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/main/java/org/apache/hadoop/yarn/server/router/webapp/RouterWebServices.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/main/java/org/apache/hadoop/yarn/server/router/webapp/RouterWebServices.java index ce45f21560c..b2ef33f5963 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/main/java/org/apache/hadoop/yarn/server/router/webapp/RouterWebServices.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/main/java/org/apache/hadoop/yarn/server/router/webapp/RouterWebServices.java @@ -465,11 +465,12 @@ public AppActivitiesInfo getAppActivities(@Context HttpServletRequest hsr, @QueryParam(RMWSConsts.REQUEST_PRIORITIES) Set requestPriorities, @QueryParam(RMWSConsts.ALLOCATION_REQUEST_IDS) Set allocationRequestIds, - @QueryParam(RMWSConsts.GROUP_BY) String groupBy) { + @QueryParam(RMWSConsts.GROUP_BY) String groupBy, + @QueryParam(RMWSConsts.LIMIT) String limit) { init(); RequestInterceptorChainWrapper pipeline = getInterceptorChain(hsr); return pipeline.getRootInterceptor().getAppActivities(hsr, appId, time, - requestPriorities, allocationRequestIds, groupBy); + requestPriorities, allocationRequestIds, groupBy, limit); } @GET diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/test/java/org/apache/hadoop/yarn/server/router/webapp/BaseRouterWebServicesTest.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/test/java/org/apache/hadoop/yarn/server/router/webapp/BaseRouterWebServicesTest.java index 535c579a85d..948ffce8405 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/test/java/org/apache/hadoop/yarn/server/router/webapp/BaseRouterWebServicesTest.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/test/java/org/apache/hadoop/yarn/server/router/webapp/BaseRouterWebServicesTest.java @@ -181,7 +181,7 @@ protected ActivitiesInfo getActivities(String user) protected AppActivitiesInfo getAppActivities(String user) throws IOException, InterruptedException { return routerWebService.getAppActivities( - createHttpServletRequest(user), null, null, null, null, null); + createHttpServletRequest(user), null, null, null, null, null, null); } protected ApplicationStatisticsInfo getAppStatistics(String user) diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/test/java/org/apache/hadoop/yarn/server/router/webapp/MockRESTRequestInterceptor.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/test/java/org/apache/hadoop/yarn/server/router/webapp/MockRESTRequestInterceptor.java index b3e18a92061..e8a34b149c4 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/test/java/org/apache/hadoop/yarn/server/router/webapp/MockRESTRequestInterceptor.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/test/java/org/apache/hadoop/yarn/server/router/webapp/MockRESTRequestInterceptor.java @@ -141,7 +141,7 @@ public ActivitiesInfo getActivities(HttpServletRequest hsr, String nodeId, @Override public AppActivitiesInfo getAppActivities(HttpServletRequest hsr, String appId, String time, Set requestPriorities, - Set allocationRequestIds, String groupBy) { + Set allocationRequestIds, String groupBy, String limit) { return new AppActivitiesInfo(); } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/test/java/org/apache/hadoop/yarn/server/router/webapp/PassThroughRESTRequestInterceptor.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/test/java/org/apache/hadoop/yarn/server/router/webapp/PassThroughRESTRequestInterceptor.java index 400bf714a86..c6b9a73414a 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/test/java/org/apache/hadoop/yarn/server/router/webapp/PassThroughRESTRequestInterceptor.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/test/java/org/apache/hadoop/yarn/server/router/webapp/PassThroughRESTRequestInterceptor.java @@ -169,9 +169,9 @@ public ActivitiesInfo getActivities(HttpServletRequest hsr, String nodeId, @Override public AppActivitiesInfo getAppActivities(HttpServletRequest hsr, String appId, String time, Set requestPriorities, - Set allocationRequestIds, String groupBy) { + Set allocationRequestIds, String groupBy, String limit) { return getNextInterceptor().getAppActivities(hsr, appId, time, - requestPriorities, allocationRequestIds, groupBy); + requestPriorities, allocationRequestIds, groupBy, limit); } @Override