From e5ba04a31d76ba57e6c884b187a6508c27fb5717 Mon Sep 17 00:00:00 2001 From: Prabhu Joseph Date: Tue, 16 Jun 2020 21:52:32 +0530 Subject: [PATCH] YARN-10319. Record Last N Scheduler Activities from ActivitiesManager Signed-off-by: Prabhu Joseph --- .../scheduler/activities/ActivitiesManager.java | 39 ++++++-- .../webapp/JAXBContextResolver.java | 3 +- .../server/resourcemanager/webapp/RMWSConsts.java | 5 ++ .../webapp/RMWebServiceProtocol.java | 14 +++ .../resourcemanager/webapp/RMWebServices.java | 59 ++++++++++++ .../webapp/dao/SchedulerActivitiesInfo.java | 52 +++++++++++ .../webapp/ActivitiesTestUtils.java | 2 + .../TestRMWebServicesSchedulerActivities.java | 100 +++++++++++++++++++++ .../webapp/DefaultRequestInterceptorREST.java | 10 +++ .../router/webapp/FederationInterceptorREST.java | 7 ++ .../server/router/webapp/RouterWebServices.java | 19 ++++ .../router/webapp/BaseRouterWebServicesTest.java | 7 ++ .../router/webapp/MockRESTRequestInterceptor.java | 7 ++ .../webapp/PassThroughRESTRequestInterceptor.java | 8 ++ .../src/site/markdown/ResourceManagerRest.md | 20 +++++ 15 files changed, 346 insertions(+), 6 deletions(-) create mode 100644 hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/SchedulerActivitiesInfo.java diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/activities/ActivitiesManager.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/activities/ActivitiesManager.java index cc02ff6..ec45077 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/activities/ActivitiesManager.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/activities/ActivitiesManager.java @@ -20,6 +20,7 @@ import com.google.common.annotations.VisibleForTesting; import com.google.common.collect.Lists; +import org.apache.commons.lang3.tuple.Pair; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.yarn.api.records.Resource; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.ResourceScheduler; @@ -44,8 +45,10 @@ import org.apache.hadoop.yarn.util.SystemClock; import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.ConcurrentLinkedDeque; import java.util.concurrent.ConcurrentLinkedQueue; import java.util.concurrent.ConcurrentMap; +import java.util.concurrent.atomic.AtomicInteger; import java.util.List; import java.util.Set; import java.util.*; @@ -75,7 +78,7 @@ appsAllocation; @VisibleForTesting ConcurrentMap> completedAppAllocations; - private boolean recordNextAvailableNode = false; + private AtomicInteger recordCount = new AtomicInteger(0); private List lastAvailableNodeActivities = null; private Thread cleanUpThread; private long activitiesCleanupIntervalMs; @@ -86,6 +89,8 @@ private final RMContext rmContext; private volatile boolean stopped; private ThreadLocal diagnosticCollectorManager; + private volatile ConcurrentLinkedDeque>> + lastNActivities; public ActivitiesManager(RMContext rmContext) { super(ActivitiesManager.class.getName()); @@ -102,6 +107,7 @@ public ActivitiesManager(RMContext rmContext) { if (rmContext.getYarnConfiguration() != null) { setupConfForCleanup(rmContext.getYarnConfiguration()); } + lastNActivities = new ConcurrentLinkedDeque<>(); } private void setupConfForCleanup(Configuration conf) { @@ -215,9 +221,29 @@ public ActivitiesInfo getActivitiesInfo(String nodeId, return new ActivitiesInfo(allocations, nodeId, groupBy); } + public List recordAndGetlastNActivitiesInfo( + int activitiesCount, RMWSConsts.ActivitiesGroupBy groupBy) + throws InterruptedException { + recordCount.set(activitiesCount); + while (recordCount.get() > 0) { + Thread.sleep(1); + } + Iterator>> ite = + lastNActivities.iterator(); + List outList = new ArrayList<>(); + while (ite.hasNext()) { + Pair> pair = ite.next(); + outList.add(new ActivitiesInfo(pair.getRight(), + pair.getLeft().toString(), groupBy)); + } + // reset with new activities + lastNActivities = new ConcurrentLinkedDeque<>(); + return outList; + } + public void recordNextNodeUpdateActivities(String nodeId) { if (nodeId == null) { - recordNextAvailableNode = true; + recordCount.set(1); } else { activeRecordedNodes.add(NodeId.fromString(nodeId)); } @@ -348,7 +374,7 @@ protected void serviceStop() throws Exception { } void startNodeUpdateRecording(NodeId nodeID) { - if (recordNextAvailableNode) { + if (recordCount.get() > 0) { recordNextNodeUpdateActivities(nodeID.toString()); } // Removing from activeRecordedNodes immediately is to ensure that @@ -470,14 +496,17 @@ void finishNodeUpdateRecording(NodeId nodeID, String partition) { allocation.setTimestamp(timestamp); allocation.setPartition(partition); } - if (recordNextAvailableNode) { - recordNextAvailableNode = false; + if (recordCount.get() > 0) { + recordCount.getAndDecrement(); } } if (shouldRecordThisNode(nodeID)) { recordingNodesAllocation.get().remove(nodeID); completedNodeAllocations.put(nodeID, value); + if (recordCount.get() >= 0) { + lastNActivities.add(Pair.of(nodeID, value)); + } } } // disable diagnostic collector diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/JAXBContextResolver.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/JAXBContextResolver.java index f6eb2ad..14f8e11 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/JAXBContextResolver.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/JAXBContextResolver.java @@ -57,7 +57,8 @@ public JAXBContextResolver() throws Exception { FairSchedulerQueueInfoList.class, AppTimeoutsInfo.class, AppTimeoutInfo.class, ResourceInformationsInfo.class, ActivitiesInfo.class, AppActivitiesInfo.class, - QueueAclsInfo.class, QueueAclInfo.class}; + QueueAclsInfo.class, QueueAclInfo.class, + SchedulerActivitiesInfo.class}; // these dao classes need root unwrapping final Class[] rootUnwrappedTypes = { NewApplication.class, ApplicationSubmissionContextInfo.class, diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMWSConsts.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMWSConsts.java index 30406e5..7bcc949 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMWSConsts.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMWSConsts.java @@ -81,6 +81,10 @@ /** Path for {@code RMWebServiceProtocol#getActivities}. */ public static final String SCHEDULER_ACTIVITIES = "/scheduler/activities"; + /** Path for {@code RMWebServiceProtocol#getLastNActivities}. */ + public static final String SCHEDULER_BULK_ACTIVITIES = + "/scheduler/bulkactivities"; + /** Path for {@code RMWebServiceProtocol#getAppActivities}. */ public static final String SCHEDULER_APP_ACTIVITIES = "/scheduler/app-activities/{appid}"; @@ -252,6 +256,7 @@ public static final String ACTIONS = "actions"; public static final String SUMMARIZE = "summarize"; public static final String NAME = "name"; + public static final String ACTIVITIES_COUNT = "activitiesCount"; private RMWSConsts() { // not called diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMWebServiceProtocol.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMWebServiceProtocol.java index a41208e..10eb5b4 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMWebServiceProtocol.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMWebServiceProtocol.java @@ -60,6 +60,7 @@ import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.ReservationUpdateRequestInfo; import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.ResourceInfo; import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.ResourceOptionInfo; +import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.SchedulerActivitiesInfo; import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.SchedulerTypeInfo; /** @@ -213,6 +214,19 @@ ActivitiesInfo getActivities(HttpServletRequest hsr, String nodeId, String groupBy); /** + * This method retrieve the last n activities inside scheduler and it is + * reachable by using {@link RMWSConsts#SCHEDULER_BULK_ACTIVITIES}. + * + * @param hsr the servlet request + * @param groupBy the groupBy type by which the activities should be + * aggregated. It is a QueryParam. + * @param activitiesCount number of activities + * @return last n activities + */ + SchedulerActivitiesInfo getLastNActivities(HttpServletRequest hsr, + String groupBy, int activitiesCount) throws InterruptedException; + + /** * This method retrieves all the activities for a specific app for a specific * period of time, and it is reachable by using * {@link RMWSConsts#SCHEDULER_APP_ACTIVITIES}. diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMWebServices.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMWebServices.java index dfdaba9..c607fc5 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMWebServices.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMWebServices.java @@ -197,6 +197,7 @@ import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.ResourceInfo; import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.ResourceOptionInfo; import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.SchedulerInfo; +import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.SchedulerActivitiesInfo; import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.SchedulerTypeInfo; import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.StatisticsItemInfo; import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.ConfigVersionInfo; @@ -242,6 +243,8 @@ public static final String DEFAULT_END_TIME = "-1"; public static final String DEFAULT_INCLUDE_RESOURCE = "false"; public static final String DEFAULT_SUMMARIZE = "false"; + public static final String DEFAULT_ACTIVITIES_COUNT = "10"; + public static final int MAX_ACTIVITIES_COUNT = 500; @VisibleForTesting boolean isCentralizedNodeLabelConfiguration = true; @@ -697,6 +700,7 @@ public AppsInfo getApps(@Context HttpServletRequest hsr, public ActivitiesInfo getActivities(@Context HttpServletRequest hsr, @QueryParam(RMWSConsts.NODEID) String nodeId, @QueryParam(RMWSConsts.GROUP_BY) String groupBy) { + initForReadableEndpoints(); YarnScheduler scheduler = rm.getRMContext().getScheduler(); @@ -771,6 +775,61 @@ public ActivitiesInfo getActivities(@Context HttpServletRequest hsr, } @GET + @Path(RMWSConsts.SCHEDULER_BULK_ACTIVITIES) + @Produces({ MediaType.APPLICATION_JSON + "; " + JettyUtils.UTF_8, + MediaType.APPLICATION_XML + "; " + JettyUtils.UTF_8 }) + @Override + public SchedulerActivitiesInfo getLastNActivities( + @Context HttpServletRequest hsr, + @QueryParam(RMWSConsts.GROUP_BY) String groupBy, + @QueryParam(RMWSConsts.ACTIVITIES_COUNT) + @DefaultValue(DEFAULT_ACTIVITIES_COUNT) int activitiesCount) + throws InterruptedException { + + initForReadableEndpoints(); + + YarnScheduler scheduler = rm.getRMContext().getScheduler(); + if (scheduler instanceof AbstractYarnScheduler) { + AbstractYarnScheduler abstractYarnScheduler = + (AbstractYarnScheduler) scheduler; + ActivitiesManager activitiesManager = + abstractYarnScheduler.getActivitiesManager(); + + if (activitiesManager != null) { + RMWSConsts.ActivitiesGroupBy activitiesGroupBy; + try { + activitiesGroupBy = parseActivitiesGroupBy(groupBy); + } catch (IllegalArgumentException e) { + throw new BadRequestException(e.getMessage()); + } + + List nodeList = + abstractYarnScheduler.getNodeTracker().getAllNodes(); + if (nodeList.size() == 0) { + throw new BadRequestException( + "No node manager running in the cluster"); + } + + if (activitiesCount <= 0) { + activitiesCount = Integer.parseInt(DEFAULT_ACTIVITIES_COUNT); + } + activitiesCount = Math.min(activitiesCount, MAX_ACTIVITIES_COUNT); + + List activitiesList = activitiesManager + .recordAndGetlastNActivitiesInfo(activitiesCount, + activitiesGroupBy); + SchedulerActivitiesInfo schedulerActivities = new + SchedulerActivitiesInfo(); + schedulerActivities.addAll(activitiesList); + + return schedulerActivities; + } + } + throw new BadRequestException(RMWSConsts.SCHEDULER_BULK_ACTIVITIES + + "is supported only for CapacityScheduler"); + } + + @GET @Path(RMWSConsts.SCHEDULER_APP_ACTIVITIES) @Produces({ MediaType.APPLICATION_JSON + "; " + JettyUtils.UTF_8, MediaType.APPLICATION_XML + "; " + JettyUtils.UTF_8 }) diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/SchedulerActivitiesInfo.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/SchedulerActivitiesInfo.java new file mode 100644 index 0000000..5a32438 --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/SchedulerActivitiesInfo.java @@ -0,0 +1,52 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.yarn.server.resourcemanager.webapp.dao; + + +import javax.xml.bind.annotation.XmlAccessType; +import javax.xml.bind.annotation.XmlAccessorType; +import javax.xml.bind.annotation.XmlRootElement; +import java.util.List; +import java.util.ArrayList; + +/** + * DAO object to display allocation activities. + */ +@XmlRootElement(name = "schedulerActivities") +@XmlAccessorType(XmlAccessType.FIELD) +public class SchedulerActivitiesInfo { + + private ArrayList activities = new ArrayList<>(); + + public SchedulerActivitiesInfo() { + // JAXB needs this + } + + public void add(ActivitiesInfo activitiesInfo) { + activities.add(activitiesInfo); + } + + public ArrayList getActivities() { + return activities; + } + + public void addAll(List activitiesInfoList) { + activities.addAll(activitiesInfoList); + } +} diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/ActivitiesTestUtils.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/ActivitiesTestUtils.java index 3c6db7d..5e6de96 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/ActivitiesTestUtils.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/ActivitiesTestUtils.java @@ -99,6 +99,8 @@ public static final String FN_SCHEDULER_ACT_ROOT = "activities"; + public static final String FN_SCHEDULER_BULK_ACT_ROOT = "schedulerActivities"; + private ActivitiesTestUtils(){} public static List findInAllocations(JSONObject allocationObj, diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesSchedulerActivities.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesSchedulerActivities.java index 1dd8020..c4a711d 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesSchedulerActivities.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesSchedulerActivities.java @@ -21,6 +21,7 @@ import com.sun.jersey.api.client.ClientResponse; import com.sun.jersey.api.client.WebResource; import com.sun.jersey.core.util.MultivaluedMapImpl; +import org.apache.hadoop.test.GenericTestUtils; import org.apache.hadoop.yarn.api.protocolrecords.AllocateRequest; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.activities.ActivityDiagnosticConstant; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.activities.ActivityState; @@ -72,6 +73,7 @@ import static org.apache.hadoop.yarn.server.resourcemanager.webapp.ActivitiesTestUtils.FN_SCHEDULER_ACT_NAME; import static org.apache.hadoop.yarn.server.resourcemanager.webapp.ActivitiesTestUtils.FN_SCHEDULER_ACT_ALLOCATIONS_ROOT; import static org.apache.hadoop.yarn.server.resourcemanager.webapp.ActivitiesTestUtils.FN_SCHEDULER_ACT_ROOT; +import static org.apache.hadoop.yarn.server.resourcemanager.webapp.ActivitiesTestUtils.FN_SCHEDULER_BULK_ACT_ROOT; import static org.apache.hadoop.yarn.server.resourcemanager.webapp.ActivitiesTestUtils.TOTAL_RESOURCE_INSUFFICIENT_DIAGNOSTIC_PREFIX; import static org.apache.hadoop.yarn.server.resourcemanager.webapp.ActivitiesTestUtils.UNMATCHED_PARTITION_OR_PC_DIAGNOSTIC_PREFIX; import static org.apache.hadoop.yarn.server.resourcemanager.webapp.ActivitiesTestUtils.getFirstSubNodeFromJson; @@ -1586,4 +1588,102 @@ public void testQueueSkippedBecauseOfHeadroom() throws Exception { rm.stop(); } } + + @Test(timeout=30000) + public void testSchedulerBulkActivities() throws Exception { + rm.start(); + + MockNM nm1 = new MockNM("127.0.0.1:1234", 4 * 1024, + rm.getResourceTrackerService()); + MockNM nm2 = new MockNM("127.0.0.2:1234", 4 * 1024, + rm.getResourceTrackerService()); + + nm1.registerNode(); + nm2.registerNode(); + + MockNM[] nms = new MockNM[] {nm1, nm2}; + + try { + + // Validate if response has 5 node activities + int expectedCount = 5; + RESTClient restClient = new RESTClient(5); + restClient.start(); + + sendHeartbeat(restClient, nms); + + JSONObject activitiesJson = restClient.getOutput().getJSONObject( + FN_SCHEDULER_BULK_ACT_ROOT); + Object activities = activitiesJson.get(FN_SCHEDULER_ACT_ROOT); + assertEquals("Number of activities is wrong", expectedCount, + ((JSONArray) activities).length()); + + + // Validate if response does not exceed max 500 + expectedCount = 1000; + restClient = new RESTClient(expectedCount); + restClient.start(); + + sendHeartbeat(restClient, nms); + + activitiesJson = restClient.getOutput().getJSONObject( + FN_SCHEDULER_BULK_ACT_ROOT); + activities = activitiesJson.get(FN_SCHEDULER_ACT_ROOT); + assertEquals("Max Activities Limit does not work", + RMWebServices.MAX_ACTIVITIES_COUNT, + ((JSONArray) activities).length()); + + } finally { + rm.stop(); + } + } + + class RESTClient extends Thread { + + private int expectedCount; + private boolean done = false; + private JSONObject json; + + RESTClient(int expectedCount) { + this.expectedCount = expectedCount; + } + + boolean isDone() { + return done; + } + + JSONObject getOutput() { + return json; + } + + @Override + public void run() { + WebResource r = resource(); + MultivaluedMapImpl params = new MultivaluedMapImpl(); + params.add(RMWSConsts.ACTIVITIES_COUNT, expectedCount); + + ClientResponse response = r.path("ws").path("v1").path("cluster") + .path(RMWSConsts.SCHEDULER_BULK_ACTIVITIES).queryParams(params) + .accept(MediaType.APPLICATION_JSON).get(ClientResponse.class); + + assertEquals(MediaType.APPLICATION_JSON_TYPE + "; " + + JettyUtils.UTF_8, response.getType().toString()); + json = response.getEntity(JSONObject.class); + done = true; + } + } + + private void sendHeartbeat(RESTClient restClient, MockNM[] nms) + throws Exception { + GenericTestUtils.waitFor(() -> { + try { + for (MockNM nm : nms) { + nm.nodeHeartbeat(true); + } + } catch (Exception e) { + return false; + } + return restClient.isDone(); + }, 10, 20000); + } } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/main/java/org/apache/hadoop/yarn/server/router/webapp/DefaultRequestInterceptorREST.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/main/java/org/apache/hadoop/yarn/server/router/webapp/DefaultRequestInterceptorREST.java index 90ca992..9519718 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/main/java/org/apache/hadoop/yarn/server/router/webapp/DefaultRequestInterceptorREST.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/main/java/org/apache/hadoop/yarn/server/router/webapp/DefaultRequestInterceptorREST.java @@ -60,6 +60,7 @@ import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.ReservationUpdateRequestInfo; import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.ResourceInfo; import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.ResourceOptionInfo; +import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.SchedulerActivitiesInfo; import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.SchedulerTypeInfo; import org.apache.hadoop.yarn.server.webapp.dao.AppAttemptInfo; import org.apache.hadoop.yarn.server.webapp.dao.ContainerInfo; @@ -200,6 +201,15 @@ public ActivitiesInfo getActivities(HttpServletRequest hsr, String nodeId, } @Override + public SchedulerActivitiesInfo getLastNActivities(HttpServletRequest hsr, + String groupBy, int activitiesCount) { + return RouterWebServiceUtil.genericForward(webAppAddress, hsr, + SchedulerActivitiesInfo.class, HTTPMethods.GET, + RMWSConsts.RM_WEB_SERVICE_PATH + RMWSConsts.SCHEDULER_BULK_ACTIVITIES, + null, null, getConf()); + } + + @Override public AppActivitiesInfo getAppActivities(HttpServletRequest hsr, String appId, String time, Set requestPriorities, Set allocationRequestIds, String groupBy, String limit, diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/main/java/org/apache/hadoop/yarn/server/router/webapp/FederationInterceptorREST.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/main/java/org/apache/hadoop/yarn/server/router/webapp/FederationInterceptorREST.java index b14da6c..0928106 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/main/java/org/apache/hadoop/yarn/server/router/webapp/FederationInterceptorREST.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/main/java/org/apache/hadoop/yarn/server/router/webapp/FederationInterceptorREST.java @@ -88,6 +88,7 @@ import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.ReservationUpdateRequestInfo; import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.ResourceInfo; import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.ResourceOptionInfo; +import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.SchedulerActivitiesInfo; import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.SchedulerTypeInfo; import org.apache.hadoop.yarn.server.router.RouterMetrics; import org.apache.hadoop.yarn.server.router.RouterServerUtil; @@ -1148,6 +1149,12 @@ public ActivitiesInfo getActivities(HttpServletRequest hsr, String nodeId, } @Override + public SchedulerActivitiesInfo getLastNActivities(HttpServletRequest hsr, + String groupBy, int activitiesCount) throws InterruptedException { + throw new NotImplementedException("Code is not implemented"); + } + + @Override public AppActivitiesInfo getAppActivities(HttpServletRequest hsr, String appId, String time, Set requestPriorities, Set allocationRequestIds, String groupBy, String limit, diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/main/java/org/apache/hadoop/yarn/server/router/webapp/RouterWebServices.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/main/java/org/apache/hadoop/yarn/server/router/webapp/RouterWebServices.java index 4c694fb..be677f3 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/main/java/org/apache/hadoop/yarn/server/router/webapp/RouterWebServices.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/main/java/org/apache/hadoop/yarn/server/router/webapp/RouterWebServices.java @@ -83,6 +83,7 @@ import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.ReservationUpdateRequestInfo; import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.ResourceInfo; import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.ResourceOptionInfo; +import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.SchedulerActivitiesInfo; import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.SchedulerTypeInfo; import org.apache.hadoop.yarn.server.router.Router; import org.apache.hadoop.yarn.server.webapp.dao.ContainerInfo; @@ -95,6 +96,7 @@ import com.google.inject.Inject; import com.google.inject.Singleton; +import static org.apache.hadoop.yarn.server.resourcemanager.webapp.RMWebServices.DEFAULT_ACTIVITIES_COUNT; import static org.apache.hadoop.yarn.server.resourcemanager.webapp.RMWebServices.DEFAULT_SUMMARIZE; /** @@ -458,6 +460,23 @@ public ActivitiesInfo getActivities(@Context HttpServletRequest hsr, } @GET + @Path(RMWSConsts.SCHEDULER_BULK_ACTIVITIES) + @Produces({ MediaType.APPLICATION_JSON + "; " + JettyUtils.UTF_8, + MediaType.APPLICATION_XML + "; " + JettyUtils.UTF_8 }) + @Override + public SchedulerActivitiesInfo getLastNActivities( + @Context HttpServletRequest hsr, + @QueryParam(RMWSConsts.GROUP_BY) String groupBy, + @QueryParam(RMWSConsts.ACTIVITIES_COUNT) + @DefaultValue(DEFAULT_ACTIVITIES_COUNT) int activitiesCount) + throws InterruptedException { + init(); + RequestInterceptorChainWrapper pipeline = getInterceptorChain(hsr); + return pipeline.getRootInterceptor().getLastNActivities(hsr, groupBy, + activitiesCount); + } + + @GET @Path(RMWSConsts.SCHEDULER_APP_ACTIVITIES) @Produces({ MediaType.APPLICATION_JSON + "; " + JettyUtils.UTF_8, MediaType.APPLICATION_XML + "; " + JettyUtils.UTF_8 }) diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/test/java/org/apache/hadoop/yarn/server/router/webapp/BaseRouterWebServicesTest.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/test/java/org/apache/hadoop/yarn/server/router/webapp/BaseRouterWebServicesTest.java index b626a8a..3e87e75 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/test/java/org/apache/hadoop/yarn/server/router/webapp/BaseRouterWebServicesTest.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/test/java/org/apache/hadoop/yarn/server/router/webapp/BaseRouterWebServicesTest.java @@ -50,6 +50,7 @@ import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.NodeLabelsInfo; import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.NodeToLabelsInfo; import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.NodesInfo; +import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.SchedulerActivitiesInfo; import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.SchedulerTypeInfo; import org.apache.hadoop.yarn.server.router.Router; import org.apache.hadoop.yarn.server.router.webapp.RouterWebServices.RequestInterceptorChainWrapper; @@ -179,6 +180,12 @@ protected ActivitiesInfo getActivities(String user) createHttpServletRequest(user), null, null); } + protected SchedulerActivitiesInfo getLastNActivities(String user) + throws InterruptedException { + return routerWebService.getLastNActivities( + createHttpServletRequest(user), null, 0); + } + protected AppActivitiesInfo getAppActivities(String user) throws IOException, InterruptedException { return routerWebService.getAppActivities(createHttpServletRequest(user), diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/test/java/org/apache/hadoop/yarn/server/router/webapp/MockRESTRequestInterceptor.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/test/java/org/apache/hadoop/yarn/server/router/webapp/MockRESTRequestInterceptor.java index f6dbb7f..efea52b 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/test/java/org/apache/hadoop/yarn/server/router/webapp/MockRESTRequestInterceptor.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/test/java/org/apache/hadoop/yarn/server/router/webapp/MockRESTRequestInterceptor.java @@ -57,6 +57,7 @@ import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.ReservationUpdateRequestInfo; import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.ResourceInfo; import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.ResourceOptionInfo; +import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.SchedulerActivitiesInfo; import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.SchedulerTypeInfo; import org.apache.hadoop.yarn.server.webapp.dao.AppAttemptInfo; import org.apache.hadoop.yarn.server.webapp.dao.ContainerInfo; @@ -139,6 +140,12 @@ public ActivitiesInfo getActivities(HttpServletRequest hsr, String nodeId, } @Override + public SchedulerActivitiesInfo getLastNActivities(HttpServletRequest hsr, + String groupBy, int activitiesCount) throws InterruptedException{ + return new SchedulerActivitiesInfo(); + } + + @Override public AppActivitiesInfo getAppActivities(HttpServletRequest hsr, String appId, String time, Set requestPriorities, Set allocationRequestIds, String groupBy, String limit, diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/test/java/org/apache/hadoop/yarn/server/router/webapp/PassThroughRESTRequestInterceptor.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/test/java/org/apache/hadoop/yarn/server/router/webapp/PassThroughRESTRequestInterceptor.java index 55de7a4..690749c 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/test/java/org/apache/hadoop/yarn/server/router/webapp/PassThroughRESTRequestInterceptor.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/test/java/org/apache/hadoop/yarn/server/router/webapp/PassThroughRESTRequestInterceptor.java @@ -55,6 +55,7 @@ import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.ReservationUpdateRequestInfo; import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.ResourceInfo; import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.ResourceOptionInfo; +import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.SchedulerActivitiesInfo; import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.SchedulerTypeInfo; import org.apache.hadoop.yarn.server.webapp.dao.AppAttemptInfo; import org.apache.hadoop.yarn.server.webapp.dao.ContainerInfo; @@ -167,6 +168,13 @@ public ActivitiesInfo getActivities(HttpServletRequest hsr, String nodeId, } @Override + public SchedulerActivitiesInfo getLastNActivities(HttpServletRequest hsr, + String groupBy, int activitiesCount) throws InterruptedException { + return getNextInterceptor().getLastNActivities(hsr, groupBy, + activitiesCount); + } + + @Override public AppActivitiesInfo getAppActivities(HttpServletRequest hsr, String appId, String time, Set requestPriorities, Set allocationRequestIds, String groupBy, String limit, diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/ResourceManagerRest.md b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/ResourceManagerRest.md index a30221d..35985eb 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/ResourceManagerRest.md +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/ResourceManagerRest.md @@ -5745,6 +5745,26 @@ Response Body: ``` +Scheduler Bulk Activities API +-------------------------------- + + The scheduler bulk activities RESTful API can fetch scheduler activities info recorded for multiple scheduling cycle. + +### URI + + * http://rm-http-address:port/ws/v1/cluster/scheduler/bulkactivities + +### HTTP Operations Supported + + * GET + +### Query Parameters Supported + +Multiple parameters can be specified for GET operations. + + * activitiesCount - number of schecduling cycle to record with maximum of 500. + * groupBy - aggregation type of application activities, currently only support "diagnostic" with which user can query aggregated activities grouped by allocation state and diagnostic. + Application Activities API -------------------------------- -- 2.7.4 (Apple Git-66)