diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/AbstractCSQueue.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/AbstractCSQueue.java index 2f6ca5a1597..61db6e493e1 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/AbstractCSQueue.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/AbstractCSQueue.java @@ -503,8 +503,6 @@ protected void setDynamicQueueProperties( String parentTemplate = String.format("%s.%s", getParent().getQueuePath(), AutoCreatedQueueTemplate.AUTO_QUEUE_TEMPLATE_PREFIX); - parentTemplate = parentTemplate.substring(0, parentTemplate.lastIndexOf( - DOT)); Set parentNodeLabels = csContext .getCapacitySchedulerQueueManager().getConfiguredNodeLabels() .getLabelsByQueue(parentTemplate); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/AutoCreatedQueueTemplate.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/AutoCreatedQueueTemplate.java index 203ec4dcd6f..cc55e059166 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/AutoCreatedQueueTemplate.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/AutoCreatedQueueTemplate.java @@ -36,13 +36,13 @@ */ public class AutoCreatedQueueTemplate { public static final String AUTO_QUEUE_TEMPLATE_PREFIX = - AUTO_QUEUE_CREATION_V2_PREFIX + "template."; + AUTO_QUEUE_CREATION_V2_PREFIX + "template"; private static final String WILDCARD_QUEUE = "*"; private static final int MAX_WILDCARD_LEVEL = 1; private final Map templateProperties = new HashMap<>(); - public AutoCreatedQueueTemplate(Configuration configuration, + public AutoCreatedQueueTemplate(CapacitySchedulerConfiguration configuration, String queuePath) { setTemplateConfigEntries(configuration, queuePath); } @@ -50,7 +50,7 @@ public AutoCreatedQueueTemplate(Configuration configuration, @VisibleForTesting public static String getAutoQueueTemplatePrefix(String queue) { return CapacitySchedulerConfiguration.getQueuePrefix(queue) - + AUTO_QUEUE_TEMPLATE_PREFIX; + + AUTO_QUEUE_TEMPLATE_PREFIX + CapacitySchedulerConfiguration.DOT; } /** @@ -67,7 +67,7 @@ public static String getAutoQueueTemplatePrefix(String queue) { * @param conf configuration to set * @param childQueuePath child queue path used for prefixing the properties */ - public void setTemplateEntriesForChild(Configuration conf, + public void setTemplateEntriesForChild(CapacitySchedulerConfiguration conf, String childQueuePath) { if (childQueuePath.equals(ROOT)) { return; @@ -94,7 +94,7 @@ public void setTemplateEntriesForChild(Configuration conf, * yarn.scheduler.capacity.root.a.auto-queue-creation-v2.template.capacity * yarn.scheduler.capacity.root.*.auto-queue-creation-v2.template.capacity */ - private void setTemplateConfigEntries(Configuration configuration, + private void setTemplateConfigEntries(CapacitySchedulerConfiguration configuration, String queuePath) { List queuePathParts = new ArrayList<>(Arrays.asList( queuePath.split("\\."))); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacitySchedulerConfiguration.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacitySchedulerConfiguration.java index 4afa2e74192..17410d44100 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacitySchedulerConfiguration.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacitySchedulerConfiguration.java @@ -406,6 +406,7 @@ public static final String MAPPING_RULE_FORMAT_DEFAULT = MAPPING_RULE_FORMAT_LEGACY; + private ConfigurationProperties configurationProperties; /** * Different resource types supported. @@ -1043,6 +1044,14 @@ public Resource getQueueMaximumAllocation(String queue) { } } + public ConfigurationProperties getConfigurationProperties() { + if (configurationProperties == null) { + configurationProperties = new ConfigurationProperties(getProps()); + } + + return configurationProperties; + } + public long getQueueMaximumAllocationMb(String queue) { String queuePrefix = getQueuePrefix(queue); return getInt(queuePrefix + MAXIMUM_ALLOCATION_MB, (int)UNDEFINED); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/ConfigurationProperties.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/ConfigurationProperties.java new file mode 100644 index 00000000000..7157c2e57a7 --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/ConfigurationProperties.java @@ -0,0 +1,190 @@ +package org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity; + +import java.util.Arrays; +import java.util.HashMap; +import java.util.Iterator; +import java.util.List; +import java.util.Map; +import java.util.TreeMap; + +/** + * A storage to preprocess and store configuration properties for optimised + * retrieval. A node is created for every key part delimited by ".". A property + * entry is stored in a node that matches its next to last key part (which + * reduces the nodes created). For example: + * yarn.scheduler.capacity.root.max-applications 100 + * yarn.scheduler.capacity.root.state RUNNING + * 4 nodes are created: yarn -> scheduler -> capacity -> root + * root node will have the two properties set in its values. + */ +public class ConfigurationProperties { + private final Map nodes; + private static final String DELIMITER = "\\."; + + /** + * A constructor defined in order to conform to the type used by + * {@code Configuration}. It must only be called by String keys and values. + * @param props properties to store + */ + public ConfigurationProperties(Map props) { + this.nodes = new HashMap<>(); + try { + // Perform an unsafe cast in order to avoid a full iteration + // on properties. + Map newProps = new TreeMap<>((Map) props); + storePropertiesInPrefixNodes(newProps); + } catch (ClassCastException exception) { + throw new IllegalArgumentException( + "Only String keys and values are allowed"); + } + } + + /** + * Filters all properties by a prefix. The property keys are trimmed by the + * given prefix. + * @param prefix prefix to filter property keys + * @return properties matching given prefix + */ + public Map getPropertiesWithPrefix(String prefix) { + return getPropertiesWithPrefix(prefix, false); + } + + /** + * Filters all properties by a prefix. + * @param prefix prefix to filter property keys + * @param fullyQualifiedKey whether collected property keys are to be trimmed + * by the prefix, or must be kept as it is + * @return properties matching given prefix + */ + public Map getPropertiesWithPrefix( + String prefix, boolean fullyQualifiedKey) { + List propertyPrefixParts = Arrays.asList(prefix.split(DELIMITER)); + Map properties = new HashMap<>(); + String trimPrefix = fullyQualifiedKey ? "" : prefix; + + recursivelyCollectProperties(nodes, properties, + propertyPrefixParts.iterator(), trimPrefix); + + return properties; + } + + /** + * Collects properties stored in all nodes that match the given prefix. + * @param childNodes children to consider when collecting properties + * @param properties aggregated property storage + * @param prefixParts prefix parts split by delimiter + * @param trimPrefix a string that needs to be trimmed from the collected + * property, empty if the key must be kept as it is + */ + private void recursivelyCollectProperties( + Map childNodes, Map properties, + Iterator prefixParts, String trimPrefix) { + if (prefixParts.hasNext()) { + String prefix = prefixParts.next(); + PrefixNode candidate = childNodes.get(prefix); + + if (candidate != null) { + if (!prefixParts.hasNext()) { + copyProperties(properties, trimPrefix, candidate.getValues()); + } + recursivelyCollectProperties(candidate.getChildren(), properties, + prefixParts, trimPrefix); + } + } else { + for (Map.Entry child : childNodes.entrySet()) { + copyProperties(properties, trimPrefix, child.getValue().getValues()); + recursivelyCollectProperties(child.getValue().getChildren(), + properties, prefixParts, trimPrefix); + } + } + } + + + /** + * Copy properties stored in a node to an aggregated property storage. + * @param copyTo property storage that collects processed properties stored + * in nodes + * @param trimPrefix a string that needs to be trimmed from the collected + * property, empty if the key must be kept as it is + * @param copyFrom properties stored in a node + */ + private void copyProperties( + Map copyTo, String trimPrefix, + Map copyFrom) { + for (Map.Entry configEntry : copyFrom.entrySet()) { + String key = configEntry.getKey(); + String prefixToTrim = trimPrefix; + + if (!trimPrefix.isEmpty()) { + if (!key.equals(trimPrefix)) { + prefixToTrim += CapacitySchedulerConfiguration.DOT; + } + key = configEntry.getKey().substring(prefixToTrim.length()); + } + + copyTo.put(key, configEntry.getValue()); + } + } + + /** + * Stores the given properties in the correct node. + * @param props properties that need to be stored + */ + private void storePropertiesInPrefixNodes(Map props) { + for (Map.Entry prop : props.entrySet()) { + String[] propertyKeyParts = prop.getKey().split(DELIMITER); + if (propertyKeyParts.length > 0) { + PrefixNode node = findOrCreatePrefixNode(nodes, + Arrays.asList(propertyKeyParts).iterator()); + node.getValues().put(prop.getKey(), prop.getValue()); + } + } + } + + /** + * Finds the node that matches the whole key or create it, if it does not + * exist. + * @param children child nodes on current level + * @param propertyKeyParts a property key split by delimiter + * @return the last node + */ + private PrefixNode findOrCreatePrefixNode( + Map children, Iterator propertyKeyParts) { + String prefix = propertyKeyParts.next(); + PrefixNode candidate = children.get(prefix); + if (candidate == null) { + candidate = new PrefixNode(); + children.put(prefix, candidate); + } + + if (!propertyKeyParts.hasNext()) { + return candidate; + } + + return findOrCreatePrefixNode(candidate.getChildren(), + propertyKeyParts); + } + + + /** + * A node that represents a prefix part. For example: + * yarn.scheduler consists of a "yarn" and a "scheduler" node. + */ + private static class PrefixNode { + private final Map values; + private final Map children; + + PrefixNode() { + this.values = new HashMap<>(); + this.children = new HashMap<>(); + } + + public Map getValues() { + return values; + } + + public Map getChildren() { + return children; + } + } +} diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestAutoCreatedQueueTemplate.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestAutoCreatedQueueTemplate.java index 2763af026a6..a10d32cdeeb 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestAutoCreatedQueueTemplate.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestAutoCreatedQueueTemplate.java @@ -122,6 +122,7 @@ public void testRootTemplate() { private String getTemplateKey(String queuePath, String entryKey) { return CapacitySchedulerConfiguration.getQueuePrefix(queuePath) - + AutoCreatedQueueTemplate.AUTO_QUEUE_TEMPLATE_PREFIX + entryKey; + + AutoCreatedQueueTemplate.AUTO_QUEUE_TEMPLATE_PREFIX + + CapacitySchedulerConfiguration.DOT + entryKey; } } \ No newline at end of file diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestConfigurationProperties.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestConfigurationProperties.java new file mode 100644 index 00000000000..81a69658c84 --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestConfigurationProperties.java @@ -0,0 +1,84 @@ +package org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity; + + +import org.junit.Assert; +import org.junit.BeforeClass; +import org.junit.Test; + +import java.util.HashMap; +import java.util.Map; + +public class TestConfigurationProperties { + private static final Map PROPERTIES = new HashMap<>(); + + @BeforeClass + public static void setUpClass() throws Exception { + PROPERTIES.put("test.test1.test2.test3", "TEST_VALUE_1"); + PROPERTIES.put("test.test1", "TEST_VALUE_2"); + PROPERTIES.put("test.test1.test2", "TEST_VALUE_3"); + PROPERTIES.put("test.test1.test2.test4", "TEST_VALUE_3_1"); + PROPERTIES.put("test.test1.test2.test4.test5", "TEST_VALUE_3_2"); + PROPERTIES.put("test", "TEST_VALUE_4"); + PROPERTIES.put("test2", "TEST_VALUE_5"); + PROPERTIES.put("test2.test3", "TEST_VALUE_5"); + } + + @Test + public void testGetPropertiesWithPrefix() { + ConfigurationProperties configurationProperties = + new ConfigurationProperties(PROPERTIES); + + Map props = configurationProperties + .getPropertiesWithPrefix("test.test1.test2"); + + Assert.assertEquals(4, props.size()); + Assert.assertTrue(props.containsKey("")); + Assert.assertTrue(props.containsKey("test4")); + Assert.assertTrue(props.containsKey("test3")); + Assert.assertTrue(props.containsKey("test4.test5")); + + Map propsWithTestPrefix = configurationProperties + .getPropertiesWithPrefix("test"); + + Assert.assertEquals(6, propsWithTestPrefix.size()); + Assert.assertTrue(propsWithTestPrefix.containsKey("")); + Assert.assertTrue(propsWithTestPrefix.containsKey("test1.test2.test3")); + Assert.assertTrue(propsWithTestPrefix.containsKey("test1")); + Assert.assertTrue(propsWithTestPrefix.containsKey("test1.test2")); + Assert.assertTrue(propsWithTestPrefix.containsKey("test1.test2.test4")); + Assert.assertTrue(propsWithTestPrefix.containsKey( + "test1.test2.test4.test5")); + } + + @Test + public void testGetPropertiesWithFullyQualifiedName() { + ConfigurationProperties configurationProperties = + new ConfigurationProperties(PROPERTIES); + + Map props = configurationProperties + .getPropertiesWithPrefix("test.test1.test2", true); + + Assert.assertEquals(4, props.size()); + Assert.assertTrue(props.containsKey("test.test1.test2.test3")); + Assert.assertTrue(props.containsKey("test.test1.test2")); + Assert.assertTrue(props.containsKey("test.test1.test2.test4.test5")); + Assert.assertTrue(props.containsKey("test.test1.test2.test4")); + } + + @Test + public void testGetPropertiesWithPrefixEmptyResult() { + ConfigurationProperties configurationProperties = + new ConfigurationProperties(PROPERTIES); + + Map propsEmptyPrefix = configurationProperties + .getPropertiesWithPrefix(""); + Map propsLongPrefix = configurationProperties + .getPropertiesWithPrefix("test.test1.test2.test4.test5.test6"); + Map propsNonExistingRootPrefix = configurationProperties + .getPropertiesWithPrefix("test3"); + + Assert.assertEquals(0, propsEmptyPrefix.size()); + Assert.assertEquals(0, propsLongPrefix.size()); + Assert.assertEquals(0, propsNonExistingRootPrefix.size()); + } +} \ No newline at end of file diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesCapacitySchedDynamicConfig.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesCapacitySchedDynamicConfig.java index 6938519c736..b745dc34632 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesCapacitySchedDynamicConfig.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesCapacitySchedDynamicConfig.java @@ -279,8 +279,8 @@ public void testSchedulerResponseWeightModeWithAutoCreatedQueues() YarnConfiguration.MEMORY_CONFIGURATION_STORE); config.setInt(CapacitySchedulerConfiguration .getQueuePrefix("root.autoParent1") + - AutoCreatedQueueTemplate.AUTO_QUEUE_TEMPLATE_PREFIX + - "maximum-applications", 300); + AutoCreatedQueueTemplate.AUTO_QUEUE_TEMPLATE_PREFIX + + CapacitySchedulerConfiguration.DOT + "maximum-applications", 300); initResourceManager(config); initAutoQueueHandler();