diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacitySchedulerConfiguration.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacitySchedulerConfiguration.java index d0ee25df300..a3414539a22 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacitySchedulerConfiguration.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacitySchedulerConfiguration.java @@ -923,6 +923,8 @@ public void setPriorityAcls(String queue, Priority priority, trimmedQueueNames.add(s.trim()); } queues = trimmedQueueNames.toArray(new String[0]); + } else { + queues = new String[0]; } LOG.debug("CSConf - getQueues: queuePrefix={}, queues={}", diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/MutableCSConfigurationProvider.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/MutableCSConfigurationProvider.java index 5f262be83d3..48cd6b4ed53 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/MutableCSConfigurationProvider.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/MutableCSConfigurationProvider.java @@ -68,11 +68,20 @@ public MutableCSConfigurationProvider(RMContext rmContext) { this.rmContext = rmContext; } + // Unit test can overwrite this method + protected Configuration getInitSchedulerConfig() { + Configuration initialSchedConf = new Configuration(false); + initialSchedConf.addResource(YarnConfiguration.CS_CONFIGURATION_FILE); + + return initialSchedConf; + } + @Override public void init(Configuration config) throws IOException { this.confStore = YarnConfigurationStoreFactory.getStore(config); - Configuration initialSchedConf = new Configuration(false); - initialSchedConf.addResource(YarnConfiguration.CS_CONFIGURATION_FILE); + + Configuration initialSchedConf = getInitSchedulerConfig(); + this.schedConf = new Configuration(false); // We need to explicitly set the key-values in schedConf, otherwise // these configuration keys cannot be deleted when @@ -226,12 +235,14 @@ public void reloadConfigurationFromStore() throws Exception { } } - private List getSiblingQueues(String queuePath, Configuration conf) { + private List getSiblingQueues(String queuePath, + CapacitySchedulerConfiguration conf) { + List list = new ArrayList<>(); String parentQueue = queuePath.substring(0, queuePath.lastIndexOf('.')); - String childQueuesKey = CapacitySchedulerConfiguration.PREFIX + - parentQueue + CapacitySchedulerConfiguration.DOT + - CapacitySchedulerConfiguration.QUEUES; - return new ArrayList<>(conf.getStringCollection(childQueuesKey)); + for (String q : conf.getQueues(parentQueue)) { + list.add(q); + } + return list; } private Map constructKeyValueConfUpdate( diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/TestMutableCSConfigurationProvider.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/TestMutableCSConfigurationProvider.java index 0c9a3120566..bc8ef7cff88 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/TestMutableCSConfigurationProvider.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/TestMutableCSConfigurationProvider.java @@ -217,6 +217,47 @@ public void testHDFSBackedProvider() throws Exception { } + @Test + public void testAddRemoveQueueWithSpacesInConfig() throws Exception { + cs = mock(CapacityScheduler.class); + rmContext = mock(RMContext.class); + when(rmContext.getScheduler()).thenReturn(cs); + when(cs.getConfiguration()).thenReturn( + new CapacitySchedulerConfiguration()); + adminService = mock(AdminService.class); + when(rmContext.getRMAdminService()).thenReturn(adminService); + + CapacitySchedulerConfiguration csConf = + new CapacitySchedulerConfiguration(); + csConf.setQueues(CapacitySchedulerConfiguration.ROOT, + new String[] {" a , b, c" }); + + final String a = CapacitySchedulerConfiguration.ROOT + ".a"; + final String b = CapacitySchedulerConfiguration.ROOT + ".b"; + final String c = CapacitySchedulerConfiguration.ROOT + ".c"; + csConf.setCapacity(a, 0); + csConf.setCapacity(b, 50); + csConf.setCapacity(c, 50); + + confProvider = new MutableCSConfigurationProvider(rmContext) { + @Override + protected Configuration getInitSchedulerConfig() { + return csConf; + } + }; + + Configuration conf = new Configuration(); + conf.set(YarnConfiguration.SCHEDULER_CONFIGURATION_STORE_CLASS, + YarnConfiguration.MEMORY_CONFIGURATION_STORE); + confProvider.init(conf); + + SchedConfUpdateInfo update = new SchedConfUpdateInfo(); + update.getRemoveQueueInfo().add("root.a"); + + confProvider.logAndApplyMutation(UserGroupInformation.getCurrentUser(), + update); + } + private void writeConf(Configuration conf, String storePath) throws IOException { FileSystem fileSystem = FileSystem.get(new Configuration(conf)); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesConfigurationMutation.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesConfigurationMutation.java index 34b7c1225c2..a37ad7f6671 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesConfigurationMutation.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesConfigurationMutation.java @@ -24,6 +24,7 @@ import com.sun.jersey.api.client.WebResource; import com.sun.jersey.guice.spi.container.servlet.GuiceContainer; import com.sun.jersey.test.framework.WebAppDescriptor; +import com.sun.tools.javac.util.List; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.security.UserGroupInformation; import org.apache.hadoop.yarn.api.records.QueueState; @@ -61,6 +62,7 @@ import static org.junit.Assert.assertNull; import static org.junit.Assert.assertNotNull; import static org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacitySchedulerConfiguration.ORDERING_POLICY; +import static org.junit.Assert.assertTrue; /** * Test scheduler configuration mutation via REST API. @@ -545,7 +547,8 @@ public void testRemoveParentQueue() throws Exception { CapacitySchedulerConfiguration newCSConf = ((CapacityScheduler) rm.getResourceScheduler()).getConfiguration(); assertEquals(2, newCSConf.getQueues("root").length); - assertNull(newCSConf.getQueues("root.c")); + String[] queueC = newCSConf.getQueues("root.c"); + assertTrue(queueC == null || queueC.length == 0); } @Test