commit 9091ab11d0afbc080ab3fb122eb409bfb60a8099 Author: Wangda Tan Date: Fri Nov 20 09:39:11 2020 -0800 COMPX-4764 Change-Id: I8bc275bb740db6b9594e75e1dad69c340ba734c1 diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/MutableCSConfigurationProvider.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/MutableCSConfigurationProvider.java index 5f262be83d3..5246921714a 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/MutableCSConfigurationProvider.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/MutableCSConfigurationProvider.java @@ -18,12 +18,10 @@ package org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.conf; -import org.apache.hadoop.thirdparty.com.google.common.annotations.VisibleForTesting; -import org.apache.hadoop.thirdparty.com.google.common.base.Joiner; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.security.UserGroupInformation; +import org.apache.hadoop.thirdparty.com.google.common.annotations.VisibleForTesting; +import org.apache.hadoop.thirdparty.com.google.common.base.Joiner; import org.apache.hadoop.yarn.conf.YarnConfiguration; import org.apache.hadoop.yarn.server.resourcemanager.RMContext; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.ConfigurationMutationACLPolicy; @@ -33,6 +31,8 @@ import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.conf.YarnConfigurationStore.LogMutation; import org.apache.hadoop.yarn.webapp.dao.QueueConfigInfo; import org.apache.hadoop.yarn.webapp.dao.SchedConfUpdateInfo; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; import java.io.IOException; import java.util.ArrayList; @@ -68,11 +68,20 @@ public MutableCSConfigurationProvider(RMContext rmContext) { this.rmContext = rmContext; } + // Unit test can overwrite this method + protected Configuration getInitSchedulerConfig() { + Configuration initialSchedConf = new Configuration(false); + initialSchedConf.addResource(YarnConfiguration.CS_CONFIGURATION_FILE); + + return initialSchedConf; + } + @Override public void init(Configuration config) throws IOException { this.confStore = YarnConfigurationStoreFactory.getStore(config); - Configuration initialSchedConf = new Configuration(false); - initialSchedConf.addResource(YarnConfiguration.CS_CONFIGURATION_FILE); + + Configuration initialSchedConf = getInitSchedulerConfig(); + this.schedConf = new Configuration(false); // We need to explicitly set the key-values in schedConf, otherwise // these configuration keys cannot be deleted when @@ -226,12 +235,13 @@ public void reloadConfigurationFromStore() throws Exception { } } - private List getSiblingQueues(String queuePath, Configuration conf) { + private List getSiblingQueues(String queuePath, CapacitySchedulerConfiguration conf) { + List list = new ArrayList<>(); String parentQueue = queuePath.substring(0, queuePath.lastIndexOf('.')); - String childQueuesKey = CapacitySchedulerConfiguration.PREFIX + - parentQueue + CapacitySchedulerConfiguration.DOT + - CapacitySchedulerConfiguration.QUEUES; - return new ArrayList<>(conf.getStringCollection(childQueuesKey)); + for (String q : conf.getQueues(parentQueue)) { + list.add(q); + } + return list; } private Map constructKeyValueConfUpdate( diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/TestMutableCSConfigurationProvider.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/TestMutableCSConfigurationProvider.java index 0c9a3120566..67b10d68247 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/TestMutableCSConfigurationProvider.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/TestMutableCSConfigurationProvider.java @@ -217,6 +217,44 @@ public void testHDFSBackedProvider() throws Exception { } + @Test + public void testAddRemoveQueueWithSpacesInConfig() throws Exception { + cs = mock(CapacityScheduler.class); + rmContext = mock(RMContext.class); + when(rmContext.getScheduler()).thenReturn(cs); + when(cs.getConfiguration()).thenReturn( + new CapacitySchedulerConfiguration()); + adminService = mock(AdminService.class); + when(rmContext.getRMAdminService()).thenReturn(adminService); + + CapacitySchedulerConfiguration csConf = new CapacitySchedulerConfiguration(); + csConf.setQueues(CapacitySchedulerConfiguration.ROOT, new String[] {" a , b, c"}); + + final String A = CapacitySchedulerConfiguration.ROOT + ".a"; + final String B = CapacitySchedulerConfiguration.ROOT + ".b"; + final String C = CapacitySchedulerConfiguration.ROOT + ".c"; + csConf.setCapacity(A, 0); + csConf.setCapacity(B, 50); + csConf.setCapacity(C, 50); + + confProvider = new MutableCSConfigurationProvider(rmContext) { + @Override + protected Configuration getInitSchedulerConfig() { + return csConf; + } + }; + + Configuration conf = new Configuration(); + conf.set(YarnConfiguration.SCHEDULER_CONFIGURATION_STORE_CLASS, + YarnConfiguration.MEMORY_CONFIGURATION_STORE); + confProvider.init(conf); + + SchedConfUpdateInfo update = new SchedConfUpdateInfo(); + update.getRemoveQueueInfo().add("root.a"); + + confProvider.logAndApplyMutation(UserGroupInformation.getCurrentUser(), update); + } + private void writeConf(Configuration conf, String storePath) throws IOException { FileSystem fileSystem = FileSystem.get(new Configuration(conf));