From ad38d7a5ccddff3a53bfc866b8f7115eefc5814b Mon Sep 17 00:00:00 2001 From: Prabhu Joseph Date: Tue, 28 Jan 2020 23:25:36 +0530 Subject: [PATCH] YARN-10109. Allow stop and convert from leaf to parent queue in a single Mutation API call --- .../apache/hadoop/yarn/api/records/QueueState.java | 7 ++++- .../capacity/CapacitySchedulerConfigValidator.java | 18 +++++------- .../TestRMWebServicesConfigurationMutation.java | 33 ++++++++++++++++++++++ 3 files changed, 46 insertions(+), 12 deletions(-) diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/QueueState.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/QueueState.java index 86fd8b5..4258c64 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/QueueState.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/QueueState.java @@ -33,6 +33,7 @@ * {@link #DRAINING} - not accepting new application submissions * and waiting for applications finish. * + *
  • {@link #UNDEFINED} - not defined.
  • * * * @see QueueInfo @@ -54,5 +55,9 @@ /** * Running - normal operation. */ - RUNNING + RUNNING, + /** + * Undefined - not defined. + */ + UNDEFINED } \ No newline at end of file diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacitySchedulerConfigValidator.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacitySchedulerConfigValidator.java index 525ea43..ea0af9e 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacitySchedulerConfigValidator.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacitySchedulerConfigValidator.java @@ -122,18 +122,13 @@ public static void validateQueueHierarchy(Map queues, String queueName = e.getKey(); CSQueue oldQueue = e.getValue(); CSQueue newQueue = newQueues.get(queueName); + String configPrefix = newConf.getQueuePrefix( + oldQueue.getQueuePath()); + String state = newConf.get(configPrefix + "state", + QueueState.UNDEFINED.name()); + QueueState newQueueState = QueueState.valueOf(state); if (null == newQueue) { // old queue doesn't exist in the new XML - String configPrefix = newConf.getQueuePrefix( - oldQueue.getQueuePath()); - QueueState newQueueState = null; - try { - newQueueState = QueueState.valueOf( - newConf.get(configPrefix + "state")); - } catch (Exception ex) { - LOG.warn("Not a valid queue state for queue " - + oldQueue.getQueuePath()); - } if (oldQueue.getState() == QueueState.STOPPED || newQueueState == QueueState.STOPPED) { LOG.info("Deleting Queue " + queueName + ", as it is not" @@ -169,7 +164,8 @@ public static void validateQueueHierarchy(Map queues, + " is set to true"); } else if (oldQueue instanceof LeafQueue && newQueue instanceof ParentQueue) { - if (oldQueue.getState() == QueueState.STOPPED) { + if (oldQueue.getState() == QueueState.STOPPED || + newQueueState == QueueState.STOPPED) { LOG.info("Converting the leaf queue: " + oldQueue.getQueuePath() + " to parent queue."); } else{ diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesConfigurationMutation.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesConfigurationMutation.java index b92fbfb..4bb50ea 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesConfigurationMutation.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesConfigurationMutation.java @@ -491,6 +491,39 @@ public void testStopWithRemoveQueue() throws Exception { } @Test + public void testStopWithConvertLeafToParentQueue() throws Exception { + WebResource r = resource(); + ClientResponse response; + + // Set state of queues to STOPPED. + SchedConfUpdateInfo updateInfo = new SchedConfUpdateInfo(); + Map stoppedParam = new HashMap<>(); + stoppedParam.put(CapacitySchedulerConfiguration.STATE, + QueueState.STOPPED.toString()); + QueueConfigInfo stoppedInfo = new QueueConfigInfo("root.b", + stoppedParam); + updateInfo.getUpdateQueueInfo().add(stoppedInfo); + + Map b1Capacity = new HashMap<>(); + b1Capacity.put(CapacitySchedulerConfiguration.CAPACITY, "100"); + QueueConfigInfo b1 = new QueueConfigInfo("root.b.b1", b1Capacity); + updateInfo.getAddQueueInfo().add(b1); + + response = r.path("ws").path("v1").path("cluster") + .path("scheduler-conf").queryParam("user.name", userName) + .accept(MediaType.APPLICATION_JSON) + .entity(YarnWebServiceUtils.toJson(updateInfo, + SchedConfUpdateInfo.class), MediaType.APPLICATION_JSON) + .put(ClientResponse.class); + + assertEquals(Status.OK.getStatusCode(), response.getStatus()); + CapacitySchedulerConfiguration newCSConf = + ((CapacityScheduler) rm.getResourceScheduler()).getConfiguration(); + assertEquals(1, newCSConf.getQueues("root.b").length); + assertEquals("b1", newCSConf.getQueues("root.b")[0]); + } + + @Test public void testRemoveParentQueue() throws Exception { WebResource r = resource(); -- 2.7.4 (Apple Git-66)