diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/AutoQueueTemplatePropertiesInfo.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/AutoQueueTemplatePropertiesInfo.java new file mode 100644 index 00000000000..889c2a8c535 --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/AutoQueueTemplatePropertiesInfo.java @@ -0,0 +1,27 @@ +package org.apache.hadoop.yarn.server.resourcemanager.webapp.dao; + +import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.LeafQueueTemplateInfo.ConfItem; + +import javax.xml.bind.annotation.XmlAccessType; +import javax.xml.bind.annotation.XmlAccessorType; +import javax.xml.bind.annotation.XmlRootElement; +import java.util.ArrayList; + +@XmlRootElement +@XmlAccessorType(XmlAccessType.FIELD) +public class AutoQueueTemplatePropertiesInfo { + + private ArrayList property = + new ArrayList<>(); + + public AutoQueueTemplatePropertiesInfo() { + } + + public ArrayList getProperty() { + return property; + } + + public void add(ConfItem confItem) { + property.add(confItem); + } +} diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/CapacitySchedulerInfo.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/CapacitySchedulerInfo.java index 78e2133a90b..c94c24fdf6b 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/CapacitySchedulerInfo.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/CapacitySchedulerInfo.java @@ -27,6 +27,7 @@ import org.apache.hadoop.util.StringUtils; import org.apache.hadoop.security.authorize.AccessControlList; import org.apache.hadoop.yarn.security.AccessType; +import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.AbstractCSQueue; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CSQueue; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacityScheduler; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacitySchedulerConfiguration; @@ -62,6 +63,7 @@ protected String creationMethod; protected String autoCreationEligibility; protected String defaultNodeLabelExpression; + protected AutoQueueTemplatePropertiesInfo autoQueueTemplateProperties; @XmlTransient static final float EPSILON = 1e-8f; @@ -109,6 +111,8 @@ public CapacitySchedulerInfo(CSQueue parent, CapacityScheduler cs) { if (parent instanceof ParentQueue) { orderingPolicyInfo = ((ParentQueue) parent).getQueueOrderingPolicy() .getConfigName(); + autoQueueTemplateProperties = CapacitySchedulerInfoHelper + .getAutoCreatedTemplateProperties((ParentQueue) parent); } mode = CapacitySchedulerInfoHelper.getMode(parent); queueType = CapacitySchedulerInfoHelper.getQueueType(parent); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/CapacitySchedulerQueueInfo.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/CapacitySchedulerQueueInfo.java index 31857bb94b6..b23ec9a3255 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/CapacitySchedulerQueueInfo.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/CapacitySchedulerQueueInfo.java @@ -92,6 +92,8 @@ protected String creationMethod; protected String autoCreationEligibility; protected String defaultNodeLabelExpression; + protected AutoQueueTemplatePropertiesInfo autoQueueTemplateProperties = + new AutoQueueTemplatePropertiesInfo(); CapacitySchedulerQueueInfo() { }; @@ -174,6 +176,8 @@ if (q instanceof ParentQueue) { orderingPolicyInfo = ((ParentQueue) q).getQueueOrderingPolicy() .getConfigName(); + autoQueueTemplateProperties = CapacitySchedulerInfoHelper + .getAutoCreatedTemplateProperties((ParentQueue) q); } String configuredCapacity = conf.get( diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/helper/CapacitySchedulerInfoHelper.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/helper/CapacitySchedulerInfoHelper.java index 355528d7f1b..0e9e8c0401a 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/helper/CapacitySchedulerInfoHelper.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/helper/CapacitySchedulerInfoHelper.java @@ -23,6 +23,12 @@ import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.LeafQueue; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.ManagedParentQueue; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.ParentQueue; +import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.AutoQueueTemplatePropertiesInfo; +import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.LeafQueueTemplateInfo; +import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.LeafQueueTemplateInfo.ConfItem; + +import java.util.Map; +import java.util.stream.Collectors; /** * Helper class to describe a queue's type, its creation method and its @@ -106,4 +112,17 @@ public static String getAutoCreationEligibility(CSQueue queue) { return AUTO_CREATION_OFF; } } + + public static AutoQueueTemplatePropertiesInfo + getAutoCreatedTemplateProperties(ParentQueue parent) { + AutoQueueTemplatePropertiesInfo propertiesInfo = + new AutoQueueTemplatePropertiesInfo(); + for (Map.Entry e : + parent.getAutoCreatedQueueTemplate().getTemplateProperties() + .entrySet()) { + propertiesInfo.add(new ConfItem(e.getKey(), e.getValue())); + } + + return propertiesInfo; + } } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesCapacitySched.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesCapacitySched.java index ab3294191bf..24744ddd597 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesCapacitySched.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesCapacitySched.java @@ -369,7 +369,7 @@ private void verifyClusterScheduler(JSONObject json) throws JSONException, JSONObject info = json.getJSONObject("scheduler"); assertEquals("incorrect number of elements in: " + info, 1, info.length()); info = info.getJSONObject("schedulerInfo"); - assertEquals("incorrect number of elements in: " + info, 19, info.length()); + assertEquals("incorrect number of elements in: " + info, 20, info.length()); verifyClusterSchedulerGeneric(info.getString("type"), (float) info.getDouble("usedCapacity"), (float) info.getDouble("capacity"), @@ -424,10 +424,10 @@ private void verifyClusterSchedulerGeneric(String type, float usedCapacity, private void verifySubQueue(JSONObject info, String q, float parentAbsCapacity, float parentAbsMaxCapacity) throws JSONException, Exception { - int numExpectedElements = 34; + int numExpectedElements = 35; boolean isParentQueue = true; if (!info.has("queues")) { - numExpectedElements = 52; + numExpectedElements = 53; isParentQueue = false; } assertEquals("incorrect number of elements", numExpectedElements, info.length()); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesCapacitySchedDynamicConfig.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesCapacitySchedDynamicConfig.java index 663bb6763d9..47312c1efda 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesCapacitySchedDynamicConfig.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesCapacitySchedDynamicConfig.java @@ -21,6 +21,7 @@ import static org.junit.Assert.assertEquals; import java.io.IOException; +import java.util.ArrayList; import java.util.Arrays; import java.util.Comparator; import java.util.HashMap; @@ -40,6 +41,7 @@ import org.apache.hadoop.yarn.server.resourcemanager.MockRM; import org.apache.hadoop.yarn.server.resourcemanager.ResourceManager; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.ResourceScheduler; +import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.AutoCreatedQueueTemplate; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CSQueueUtils; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacityScheduler; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacitySchedulerConfiguration; @@ -93,6 +95,7 @@ private String queueType; private String creationMethod; private String autoCreationEligibility; + private List autoQueueTemplateProperties; public ExpectedQueueWithProperties(String path, float weight, float normalizedWeight, String queueType, String creationMethod, @@ -103,6 +106,20 @@ public ExpectedQueueWithProperties(String path, float weight, this.queueType = queueType; this.creationMethod = creationMethod; this.autoCreationEligibility = autoCreationEligibility; + this.autoQueueTemplateProperties = new ArrayList<>(); + } + + public ExpectedQueueWithProperties( + String path, float weight, float normalizedWeight, String queueType, + String creationMethod, String autoCreationEligibility, + List autoQueueTemplateProperties) { + this.path = path; + this.weight = weight; + this.normalizedWeight = normalizedWeight; + this.queueType = queueType; + this.creationMethod = creationMethod; + this.autoCreationEligibility = autoCreationEligibility; + this.autoQueueTemplateProperties = autoQueueTemplateProperties; } } @@ -260,6 +277,9 @@ public void testSchedulerResponseWeightModeWithAutoCreatedQueues() .createWeightConfigWithAutoQueueCreationEnabled(); config.set(YarnConfiguration.SCHEDULER_CONFIGURATION_STORE_CLASS, YarnConfiguration.MEMORY_CONFIGURATION_STORE); + config.setInt(CapacitySchedulerConfiguration.getQueuePrefix("root.autoParent1") + + AutoCreatedQueueTemplate.AUTO_QUEUE_TEMPLATE_PREFIX + + "maximum-applications", 300); initResourceManager(config); initAutoQueueHandler(); @@ -290,6 +310,9 @@ public void testSchedulerResponseWeightModeWithAutoCreatedQueues() new ExpectedQueueWithProperties("root", EXP_ROOT_WEIGHT_IN_WEIGHT_MODE, EXP_ROOT_WEIGHT_IN_WEIGHT_MODE, PARENT_QUEUE, STATIC_QUEUE, AUTO_CREATION_OFF); + List templateProperties = new ArrayList<>(); + templateProperties.add(new String[] {"maximum-applications", "300"}); + validateSchedulerInfo(json, "weight", expectedRootQ, new ExpectedQueueWithProperties("root.auto1", @@ -307,7 +330,8 @@ public void testSchedulerResponseWeightModeWithAutoCreatedQueues() new ExpectedQueueWithProperties("root.autoParent1", EXP_DEFAULT_WEIGHT_IN_WEIGHT_MODE, EXP_DEFAULT_WEIGHT_IN_WEIGHT_MODE / sumOfWeights, - PARENT_QUEUE, FLEXIBLE_DYNAMIC_QUEUE, AUTO_CREATION_FLEXIBLE), + PARENT_QUEUE, FLEXIBLE_DYNAMIC_QUEUE, AUTO_CREATION_FLEXIBLE, + templateProperties), new ExpectedQueueWithProperties("root.default", 10.0f, 10.0f / sumOfWeights, LEAF_QUEUE, STATIC_QUEUE, AUTO_CREATION_OFF), @@ -474,6 +498,22 @@ private void verifyQueues(String parentPath, String expectedMode, queuePath, expectedQueue.creationMethod, obj.getString("creationMethod")); + if (!expectedQueue.autoQueueTemplateProperties.isEmpty()) { + JSONArray templates = obj.getJSONObject("autoQueueTemplateProperties") + .getJSONArray("property"); + for (int j = 0; j < templates.length(); j++) { + JSONObject prop = templates.getJSONObject(j); + Assert.assertEquals("Auto creation eligible queue " + + "template key do not match for queue" + queuePath, + expectedQueue.autoQueueTemplateProperties.get(j)[0], + prop.getString("name")); + Assert.assertEquals("Auto creation eligible queue " + + "template value do not match for queue" + queuePath, + expectedQueue.autoQueueTemplateProperties.get(j)[1], + prop.getString("value")); + } + } + Assert.assertEquals("Queue auto creation eligibility does not " + "match for queue " + queuePath, expectedQueue.autoCreationEligibility,