From 2eb634d96fba41b41e8b597e80100ced4c628b0a Mon Sep 17 00:00:00 2001 From: Prabhu Joseph Date: Mon, 26 Aug 2019 13:13:24 +0530 Subject: [PATCH] YARN-9780. Allow Stop and Remove queue in single SchedulerConf Rest Api Call. Signed-off-by: Prabhu Joseph --- .../capacity/CapacitySchedulerQueueManager.java | 18 ++++++++++-- .../TestRMWebServicesConfigurationMutation.java | 32 ++++++++++++++++++++++ 2 files changed, 47 insertions(+), 3 deletions(-) diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacitySchedulerQueueManager.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacitySchedulerQueueManager.java index 7dca824..d9b3ebd 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacitySchedulerQueueManager.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacitySchedulerQueueManager.java @@ -177,7 +177,7 @@ public void reinitializeQueues(CapacitySchedulerConfiguration newConf) csContext.getRMContext().getHAServiceState() != HAServiceProtocol.HAServiceState.STANDBY) { // Ensure queue hierarchy in the new XML file is proper. - validateQueueHierarchy(queues, newQueues); + validateQueueHierarchy(queues, newQueues, newConf); } // Add new queues and delete OldQeueus only after validation. @@ -309,7 +309,8 @@ static CSQueue parseQueue( * @param newQueues new queues */ private void validateQueueHierarchy(Map queues, - Map newQueues) throws IOException { + Map newQueues, CapacitySchedulerConfiguration newConf) + throws IOException { // check that all static queues are included in the newQueues list for (Map.Entry e : queues.entrySet()) { if (!(AbstractAutoCreatedLeafQueue.class.isAssignableFrom(e.getValue() @@ -319,7 +320,18 @@ private void validateQueueHierarchy(Map queues, CSQueue newQueue = newQueues.get(queueName); if (null == newQueue) { // old queue doesn't exist in the new XML - if (oldQueue.getState() == QueueState.STOPPED) { + String configPrefix = newConf.getQueuePrefix( + oldQueue.getQueuePath()); + QueueState newQueueState = null; + try { + newQueueState = QueueState.valueOf( + newConf.get(configPrefix + "state")); + } catch (Exception ex) { + LOG.warn("Not a valid queue state for queue " + + oldQueue.getQueuePath()); + } + if (oldQueue.getState() == QueueState.STOPPED || + newQueueState == QueueState.STOPPED) { LOG.info("Deleting Queue " + queueName + ", as it is not" + " present in the modified capacity configuration xml"); } else{ diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesConfigurationMutation.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesConfigurationMutation.java index 773b4bf..769f992 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesConfigurationMutation.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesConfigurationMutation.java @@ -354,6 +354,38 @@ public void testRemoveQueue() throws Exception { assertEquals(Status.OK.getStatusCode(), response.getStatus()); CapacitySchedulerConfiguration newCSConf = ((CapacityScheduler) rm.getResourceScheduler()).getConfiguration(); + assertEquals("Failed to remove the queue", + 1, newCSConf.getQueues("root.a").length); + assertEquals("Failed to remove the right queue", + "a1", newCSConf.getQueues("root.a")[0]); + } + + @Test + public void testStopWithRemoveQueue() throws Exception { + WebResource r = resource(); + + ClientResponse response; + + // Set state of queues to STOPPED. + SchedConfUpdateInfo updateInfo = new SchedConfUpdateInfo(); + Map stoppedParam = new HashMap<>(); + stoppedParam.put(CapacitySchedulerConfiguration.STATE, + QueueState.STOPPED.toString()); + QueueConfigInfo stoppedInfo = new QueueConfigInfo("root.a.a2", + stoppedParam); + updateInfo.getUpdateQueueInfo().add(stoppedInfo); + + updateInfo.getRemoveQueueInfo().add("root.a.a2"); + response = r.path("ws").path("v1").path("cluster") + .path("scheduler-conf").queryParam("user.name", userName) + .accept(MediaType.APPLICATION_JSON) + .entity(YarnWebServiceUtils.toJson(updateInfo, + SchedConfUpdateInfo.class), MediaType.APPLICATION_JSON) + .put(ClientResponse.class); + + assertEquals(Status.OK.getStatusCode(), response.getStatus()); + CapacitySchedulerConfiguration newCSConf = + ((CapacityScheduler) rm.getResourceScheduler()).getConfiguration(); assertEquals(1, newCSConf.getQueues("root.a").length); assertEquals("a1", newCSConf.getQueues("root.a")[0]); } -- 2.7.4 (Apple Git-66)