diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/AbstractCSQueue.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/AbstractCSQueue.java index 12ce05f2791..b0912215a29 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/AbstractCSQueue.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/AbstractCSQueue.java @@ -1614,6 +1614,12 @@ void updateEffectiveResources(Resource clusterResource) { } } + public CapacitySchedulerConfiguration getTemplateConfiguration() { + return isDynamicQueue() ? new AutoCreatedQueueTemplateConfig( + csContext.getConfiguration(), getQueuePath()) : + csContext.getConfiguration(); + } + public boolean isDynamicQueue() { readLock.lock(); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/AutoCreatedQueueTemplateConfig.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/AutoCreatedQueueTemplateConfig.java new file mode 100644 index 00000000000..00804fa053c --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/AutoCreatedQueueTemplateConfig.java @@ -0,0 +1,138 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + *
+ * http://www.apache.org/licenses/LICENSE-2.0 + *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity;
+
+import org.apache.hadoop.conf.Configuration;
+
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+
+/**
+ * Represents a configuration that handles template entries based on
+ * template settings.
+ */
+public class AutoCreatedQueueTemplateConfig
+ extends CapacitySchedulerConfiguration {
+ public static final String AUTO_QUEUE_TEMPLATE_PREFIX = PREFIX +
+ AUTO_QUEUE_CREATION_V2_PREFIX + ".template.";
+ private static final String WILDCARD_QUEUE = "*";
+ private static final int MAX_WILDCARD_LEVEL = 2;
+
+ private final String queuePath;
+
+ public AutoCreatedQueueTemplateConfig(Configuration configuration,
+ String queuePath) {
+ super(configuration);
+ this.queuePath = queuePath;
+ setTemplateConfigEntries();
+ }
+
+ /**
+ * Set the template configuration entries. Explicit templates always take
+ * precedence over wildcard values. An example template precedence
+ * hierarchy of root.a.b.c from highest to lowest:
+ * yarn.scheduler.capacity.auto-queue-creation-v2.template.root.a.b.capacity
+ * yarn.scheduler.capacity.auto-queue-creation-v2.template.root.a.*.capacity
+ * yarn.scheduler.capacity.auto-queue-creation-v2.template.root.*.*.capacity
+ */
+ private void setTemplateConfigEntries() {
+ HashMap
+ * It is necessary to check property keys beforehand, to exclude configuration
+ * settings from lower queue levels (because there is no uniform way to
+ * distinguish whether a part of a key is a part of a queue path or a
+ * configuration property name). This is limited however to one-part keys
+ * (eg. capacity) or two-parts keys with an enable tag
+ * (eg. preemption.enabled)
+ * Example: for 'root.a' exclude 'root.a.b.capacity' but include
+ * 'root.a.preemption.enabled'
+ *
+ * @param propKey property key to check
+ * @return true if property is valid, false otherwise
+ */
+ private static boolean isPropValid(String propKey) {
+ String[] keyParts = propKey.split("\\.");
+ boolean isSwitchProperty = keyParts.length == 2
+ && (keyParts[1].endsWith("enabled") || keyParts[1].endsWith("enable"));
+ return keyParts.length == 1 || isSwitchProperty;
+ }
+
+}
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacitySchedulerConfiguration.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacitySchedulerConfiguration.java
index 08b38a1707e..bb3ef3a1125 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacitySchedulerConfiguration.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacitySchedulerConfiguration.java
@@ -2009,7 +2009,7 @@ public void setDefaultLifetimePerQueue(String queue, long defaultLifetime) {
AUTO_CREATE_CHILD_QUEUE_PREFIX + "enabled";
@Private
- private static final String AUTO_QUEUE_CREATION_V2_PREFIX =
+ protected static final String AUTO_QUEUE_CREATION_V2_PREFIX =
"auto-queue-creation-v2.";
@Private
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/LeafQueue.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/LeafQueue.java
index 6bf8d0a471a..84237bde255 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/LeafQueue.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/LeafQueue.java
@@ -563,8 +563,7 @@ protected void reinitialize(
public void reinitialize(
CSQueue newlyParsedQueue, Resource clusterResource)
throws IOException {
- reinitialize(newlyParsedQueue, clusterResource,
- csContext.getConfiguration());
+ reinitialize(newlyParsedQueue, clusterResource, getTemplateConfiguration());
}
@Override
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/ParentQueue.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/ParentQueue.java
index f79ffcbb33c..7f78e2adcc2 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/ParentQueue.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/ParentQueue.java
@@ -30,6 +30,7 @@
import org.apache.hadoop.thirdparty.com.google.common.collect.ImmutableList;
import org.apache.hadoop.thirdparty.com.google.common.collect.ImmutableMap;
import org.apache.commons.lang3.StringUtils;
+import org.apache.hadoop.thirdparty.com.google.common.collect.Sets;
import org.apache.hadoop.yarn.exceptions.YarnRuntimeException;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.SchedulerDynamicEditException;
import org.slf4j.Logger;
@@ -475,8 +476,8 @@ private CapacitySchedulerConfiguration getConfForAutoCreatedQueue(
String childQueuePath, boolean isLeaf) {
// Copy existing config
CapacitySchedulerConfiguration dupCSConfig =
- new CapacitySchedulerConfiguration(
- csContext.getConfiguration(), false);
+ new AutoCreatedQueueTemplateConfig(csContext.getConfiguration(),
+ childQueuePath);
if (isLeaf) {
// set to -1, to disable it
dupCSConfig.setUserLimitFactor(childQueuePath, -1);
@@ -617,13 +618,24 @@ public void reinitialize(CSQueue newlyParsedQueue,
ParentQueue newlyParsedParentQueue = (ParentQueue) newlyParsedQueue;
// Set new configs
- setupQueueConfigs(clusterResource, csContext.getConfiguration());
+ setupQueueConfigs(clusterResource, getTemplateConfiguration());
// Re-configure existing child queues and add new ones
// The CS has already checked to ensure all existing child queues are present!
Map
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity;
+
+import org.junit.Assert;
+import org.junit.Before;
+import org.junit.Test;
+
+import java.util.Map;
+
+public class TestAutoCreatedQueueTemplateConfig {
+ private static final String TEST_QUEUE_ABC = "root.a.b.c";
+ private static final String TEST_QUEUE_AB = "root.a.b";
+ private static final String TEST_QUEUE_A = "root.a";
+ private static final String ROOT = "root";
+ private CapacitySchedulerConfiguration conf;
+
+ @Before
+ public void setUp() throws Exception {
+ conf = new CapacitySchedulerConfiguration();
+ conf.setQueues("root", new String[]{"a"});
+ conf.setQueues("a", new String[]{"b"});
+ conf.setQueues("b", new String[]{"c"});
+ }
+
+ @Test
+ public void testNonWildCardTemplate() {
+ conf.set(getTemplateKey(TEST_QUEUE_AB, "capacity"), "6w");
+ AutoCreatedQueueTemplateConfig templateConfig =
+ new AutoCreatedQueueTemplateConfig(conf, TEST_QUEUE_ABC);
+
+ Assert.assertEquals("weight is not set", 6f,
+ templateConfig.getNonLabeledQueueWeight(TEST_QUEUE_ABC), 10e-6);
+
+ }
+
+ @Test
+ public void testOneLevelWildcardTemplate() {
+ conf.set(getTemplateKey("root.a.*", "capacity"), "6w");
+ AutoCreatedQueueTemplateConfig templateConfig =
+ new AutoCreatedQueueTemplateConfig(conf, TEST_QUEUE_ABC);
+
+ Assert.assertEquals("weight is not set", 6f,
+ templateConfig.getNonLabeledQueueWeight(TEST_QUEUE_ABC), 10e-6);
+
+ }
+
+ @Test
+ public void testTwoLevelsWildcardTemplate() {
+ conf.set(getTemplateKey("root.*.*", "capacity"), "6w");
+ AutoCreatedQueueTemplateConfig templateConfig =
+ new AutoCreatedQueueTemplateConfig(conf, TEST_QUEUE_ABC);
+
+ Assert.assertEquals("weight is not set", 6f,
+ templateConfig.getNonLabeledQueueWeight(TEST_QUEUE_ABC), 10e-6);
+ }
+
+ @Test
+ public void testIgnoredWhenRootWildcarded() {
+ conf.set(getTemplateKey("*", "capacity"), "6w");
+ AutoCreatedQueueTemplateConfig templateConfig =
+ new AutoCreatedQueueTemplateConfig(conf, TEST_QUEUE_A);
+
+ Assert.assertEquals("weight is set", -1f,
+ templateConfig.getNonLabeledQueueWeight(TEST_QUEUE_A), 10e-6);
+ }
+
+ @Test
+ public void testIgnoredWhenNoParent() {
+ conf.set(getTemplateKey("root", "capacity"), "6w");
+ AutoCreatedQueueTemplateConfig templateConfig =
+ new AutoCreatedQueueTemplateConfig(conf, ROOT);
+
+ Assert.assertEquals("weight is set", -1f,
+ templateConfig.getNonLabeledQueueWeight(ROOT), 10e-6);
+ }
+
+ @Test
+ public void testTemplatePrecedence() {
+ conf.set(getTemplateKey("root.a.b", "capacity"), "6w");
+ conf.set(getTemplateKey("root.a.*", "capacity"), "4w");
+ conf.set(getTemplateKey("root.*.*", "capacity"), "2w");
+
+ AutoCreatedQueueTemplateConfig templateConfig =
+ new AutoCreatedQueueTemplateConfig(conf, TEST_QUEUE_ABC);
+
+ Assert.assertEquals(
+ "explicit template does not have the highest precedence", 6f,
+ templateConfig.getNonLabeledQueueWeight(TEST_QUEUE_ABC), 10e-6);
+
+ CapacitySchedulerConfiguration newConf =
+ new CapacitySchedulerConfiguration();
+ newConf.set(getTemplateKey("root.a.*", "capacity"), "4w");
+ newConf.set(getTemplateKey("root.*.*", "capacity"), "2w");
+
+ AutoCreatedQueueTemplateConfig newTemplateConfig =
+ new AutoCreatedQueueTemplateConfig(newConf, TEST_QUEUE_ABC);
+
+ Assert.assertEquals("precedence is invalid", 4f,
+ newTemplateConfig.getNonLabeledQueueWeight(TEST_QUEUE_ABC), 10e-6);
+ }
+
+ @Test
+ public void testIgnoreLowerQueueTemplates() {
+ conf.set(getTemplateKey("root.a.b", "capacity"), "6w");
+
+ AutoCreatedQueueTemplateConfig templateConfig =
+ new AutoCreatedQueueTemplateConfig(conf, TEST_QUEUE_A);
+
+ Map