From d6a94a88e5f30b8f4ff9face5f41a6058cd52d98 Mon Sep 17 00:00:00 2001 From: Prabhu Joseph Date: Mon, 26 Aug 2019 13:13:24 +0530 Subject: [PATCH] YARN-9780. Allow Stop and Remove queue in single SchedulerConf Rest Api Call. --- .../capacity/CapacitySchedulerQueueManager.java | 12 ++++++--- .../TestRMWebServicesConfigurationMutation.java | 30 ++++++++++++++++++++++ 2 files changed, 39 insertions(+), 3 deletions(-) diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacitySchedulerQueueManager.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacitySchedulerQueueManager.java index cdde25d..902654f 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacitySchedulerQueueManager.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacitySchedulerQueueManager.java @@ -177,7 +177,7 @@ public void reinitializeQueues(CapacitySchedulerConfiguration newConf) csContext.getRMContext().getHAServiceState() != HAServiceProtocol.HAServiceState.STANDBY) { // Ensure queue hierarchy in the new XML file is proper. - validateQueueHierarchy(queues, newQueues); + validateQueueHierarchy(queues, newQueues, newConf); } // Add new queues and delete OldQeueus only after validation. @@ -309,7 +309,8 @@ static CSQueue parseQueue( * @param newQueues new queues */ private void validateQueueHierarchy(Map queues, - Map newQueues) throws IOException { + Map newQueues, CapacitySchedulerConfiguration newConf) + throws IOException { // check that all static queues are included in the newQueues list for (Map.Entry e : queues.entrySet()) { if (!(AbstractAutoCreatedLeafQueue.class.isAssignableFrom(e.getValue() @@ -319,7 +320,12 @@ private void validateQueueHierarchy(Map queues, CSQueue newQueue = newQueues.get(queueName); if (null == newQueue) { // old queue doesn't exist in the new XML - if (oldQueue.getState() == QueueState.STOPPED) { + String configPrefix = newConf.getQueuePrefix( + oldQueue.getQueuePath()); + String newQueueState = newConf.get(configPrefix+"state"); + if (oldQueue.getState() == QueueState.STOPPED || + (newQueueState !=null && + newQueueState.equals(QueueState.STOPPED.name()))) { LOG.info("Deleting Queue " + queueName + ", as it is not" + " present in the modified capacity configuration xml"); } else{ diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesConfigurationMutation.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesConfigurationMutation.java index 3e2542c..c52cdde 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesConfigurationMutation.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesConfigurationMutation.java @@ -298,6 +298,36 @@ public void testRemoveQueue() throws Exception { } @Test + public void testStopWithRemoveQueue() throws Exception { + WebResource r = resource(); + + ClientResponse response; + + // Set state of queues to STOPPED. + SchedConfUpdateInfo updateInfo = new SchedConfUpdateInfo(); + Map stoppedParam = new HashMap<>(); + stoppedParam.put(CapacitySchedulerConfiguration.STATE, + QueueState.STOPPED.toString()); + QueueConfigInfo stoppedInfo = new QueueConfigInfo("root.a.a2", + stoppedParam); + updateInfo.getUpdateQueueInfo().add(stoppedInfo); + + updateInfo.getRemoveQueueInfo().add("root.a.a2"); + response = r.path("ws").path("v1").path("cluster") + .path("scheduler-conf").queryParam("user.name", userName) + .accept(MediaType.APPLICATION_JSON) + .entity(YarnWebServiceUtils.toJson(updateInfo, + SchedConfUpdateInfo.class), MediaType.APPLICATION_JSON) + .put(ClientResponse.class); + + assertEquals(Status.OK.getStatusCode(), response.getStatus()); + CapacitySchedulerConfiguration newCSConf = + ((CapacityScheduler) rm.getResourceScheduler()).getConfiguration(); + assertEquals(1, newCSConf.getQueues("root.a").length); + assertEquals("a1", newCSConf.getQueues("root.a")[0]); + } + + @Test public void testRemoveParentQueue() throws Exception { WebResource r = resource(); -- 2.7.4 (Apple Git-66)