From 0320910fc4e5b1772b60945a960a9f1a2fba4c35 Mon Sep 17 00:00:00 2001 From: Prabhu Joseph Date: Tue, 28 Jan 2020 23:25:36 +0530 Subject: [PATCH] YARN-10109. Allow stop and convert from leaf to parent queue in a single Mutation API call --- .../capacity/CapacitySchedulerConfigValidator.java | 22 +++++++-------- .../TestRMWebServicesConfigurationMutation.java | 33 ++++++++++++++++++++++ 2 files changed, 44 insertions(+), 11 deletions(-) diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacitySchedulerConfigValidator.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacitySchedulerConfigValidator.java index 525ea43..9f967c6 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacitySchedulerConfigValidator.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacitySchedulerConfigValidator.java @@ -122,18 +122,17 @@ public static void validateQueueHierarchy(Map queues, String queueName = e.getKey(); CSQueue oldQueue = e.getValue(); CSQueue newQueue = newQueues.get(queueName); + String configPrefix = newConf.getQueuePrefix( + oldQueue.getQueuePath()); + QueueState newQueueState = null; + try { + newQueueState = QueueState.valueOf( + newConf.get(configPrefix + "state")); + } catch (Exception ex) { + // ignore the exception as the config state is optional + } if (null == newQueue) { // old queue doesn't exist in the new XML - String configPrefix = newConf.getQueuePrefix( - oldQueue.getQueuePath()); - QueueState newQueueState = null; - try { - newQueueState = QueueState.valueOf( - newConf.get(configPrefix + "state")); - } catch (Exception ex) { - LOG.warn("Not a valid queue state for queue " - + oldQueue.getQueuePath()); - } if (oldQueue.getState() == QueueState.STOPPED || newQueueState == QueueState.STOPPED) { LOG.info("Deleting Queue " + queueName + ", as it is not" @@ -169,7 +168,8 @@ public static void validateQueueHierarchy(Map queues, + " is set to true"); } else if (oldQueue instanceof LeafQueue && newQueue instanceof ParentQueue) { - if (oldQueue.getState() == QueueState.STOPPED) { + if (oldQueue.getState() == QueueState.STOPPED || + newQueueState == QueueState.STOPPED) { LOG.info("Converting the leaf queue: " + oldQueue.getQueuePath() + " to parent queue."); } else{ diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesConfigurationMutation.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesConfigurationMutation.java index b92fbfb..4bb50ea 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesConfigurationMutation.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesConfigurationMutation.java @@ -491,6 +491,39 @@ public void testStopWithRemoveQueue() throws Exception { } @Test + public void testStopWithConvertLeafToParentQueue() throws Exception { + WebResource r = resource(); + ClientResponse response; + + // Set state of queues to STOPPED. + SchedConfUpdateInfo updateInfo = new SchedConfUpdateInfo(); + Map stoppedParam = new HashMap<>(); + stoppedParam.put(CapacitySchedulerConfiguration.STATE, + QueueState.STOPPED.toString()); + QueueConfigInfo stoppedInfo = new QueueConfigInfo("root.b", + stoppedParam); + updateInfo.getUpdateQueueInfo().add(stoppedInfo); + + Map b1Capacity = new HashMap<>(); + b1Capacity.put(CapacitySchedulerConfiguration.CAPACITY, "100"); + QueueConfigInfo b1 = new QueueConfigInfo("root.b.b1", b1Capacity); + updateInfo.getAddQueueInfo().add(b1); + + response = r.path("ws").path("v1").path("cluster") + .path("scheduler-conf").queryParam("user.name", userName) + .accept(MediaType.APPLICATION_JSON) + .entity(YarnWebServiceUtils.toJson(updateInfo, + SchedConfUpdateInfo.class), MediaType.APPLICATION_JSON) + .put(ClientResponse.class); + + assertEquals(Status.OK.getStatusCode(), response.getStatus()); + CapacitySchedulerConfiguration newCSConf = + ((CapacityScheduler) rm.getResourceScheduler()).getConfiguration(); + assertEquals(1, newCSConf.getQueues("root.b").length); + assertEquals("b1", newCSConf.getQueues("root.b")[0]); + } + + @Test public void testRemoveParentQueue() throws Exception { WebResource r = resource(); -- 2.7.4 (Apple Git-66)