From 41aacc09e21245ea2d7b5cc6be0a66ef774a29fe Mon Sep 17 00:00:00 2001 From: Prabhu Joseph Date: Tue, 22 Oct 2019 21:15:10 +0530 Subject: [PATCH] YARN-9925. Disallow unsupported queue hierarchy in CapacityScheduler. CapacityScheduler currently does not support full queue path. It fails when adding queue with same name as an existing parent queue - "Queue's cannot be moved from one hierarchy to other also". Some cases are not hanlded and which fails while submission of job - "Failed to submit application to non-leaf queue". This patch completely avoids adding a queue with same name as an existing queue to fix the same. This check can be removed once CapacityScheduler supports full queue path (YARN-7621). Signed-off-by: prabhujoseph --- .../capacity/CapacitySchedulerQueueManager.java | 8 +++---- .../scheduler/capacity/TestCapacityScheduler.java | 28 ++++++++++++++++++++++ .../src/site/markdown/CapacityScheduler.md | 2 ++ 3 files changed, 34 insertions(+), 4 deletions(-) diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacitySchedulerQueueManager.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacitySchedulerQueueManager.java index 7dca824..1bb22cc 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacitySchedulerQueueManager.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacitySchedulerQueueManager.java @@ -288,10 +288,10 @@ static CSQueue parseQueue( } - if (queue instanceof LeafQueue && queues.containsKey(queueName) && queues - .get(queueName) instanceof LeafQueue) { - throw new IOException("Two leaf queues were named " + queueName - + ". Leaf queue names must be distinct"); + if (queues.containsKey(queueName)) { + throw new IOException("Two queues " + fullQueueName + " and " + + queues.get(queueName).getQueuePath() + + " named with same queue name. Queue names must be distinct"); } queues.put(queueName, queue); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacityScheduler.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacityScheduler.java index 01be51c..3a24c72 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacityScheduler.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacityScheduler.java @@ -245,6 +245,34 @@ private NodeManager registerNode(ResourceManager rm, String hostName, } @Test (timeout = 30000) + public void testDisallowUniqueQueueNames() { + CapacitySchedulerConfiguration conf = new CapacitySchedulerConfiguration(); + setupQueueConfiguration(conf); + + // Add queue root.a.b1 which conflicts with existing queue root.b.b1 + conf.setQueues(A, new String[] {"b1"}); + conf.setCapacity(A + ".b1", 100f); + + CapacityScheduler cs = new CapacityScheduler(); + cs.setConf(new YarnConfiguration()); + cs.setRMContext(resourceManager.getRMContext()); + + boolean exceptionCaught = false; + String exceptionMessage = ""; + try { + cs.init(conf); + } catch (Exception e) { + exceptionCaught = true; + exceptionMessage = e.getMessage(); + } + assertTrue("Failed to disallow unique queue names", exceptionCaught); + assertTrue(exceptionMessage.contains("Two queues root.b.b1 and root.a.b1" + + " named with same queue name. Queue names must be distinct")); + + cs.stop(); + } + + @Test (timeout = 30000) public void testConfValidation() throws Exception { CapacityScheduler scheduler = new CapacityScheduler(); scheduler.setRMContext(resourceManager.getRMContext()); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/CapacityScheduler.md b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/CapacityScheduler.md index aa137c0..3c14b82 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/CapacityScheduler.md +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/CapacityScheduler.md @@ -91,6 +91,8 @@ Configuration A given queue's children can be defined with the configuration knob: `yarn.scheduler.capacity..queues`. Children do not inherit properties directly from the parent unless otherwise noted. + Currently, CapacityScheduler does not allow two queues with same name irrespective of hierarchy or position. For example, root.a.a1 and root.b.a1 are not allowed. root.a.b.c and root.a.d.b.e are not allowed. + Here is an example with three top-level child-queues `a`, `b` and `c` and some sub-queues for `a` and `b`: ```xml -- 2.7.4 (Apple Git-66)