From 8ab3fac475572b8b608458fe43414ac46b16ca61 Mon Sep 17 00:00:00 2001 From: Prabhu Joseph Date: Tue, 25 Feb 2020 01:07:00 +0530 Subject: [PATCH] YARN-10160. Add auto queue creation related configs to RMWebService#CapacitySchedulerQueueInfo --- .../webapp/dao/CapacitySchedulerQueueInfo.java | 14 ++++ .../webapp/dao/LeafQueueTemplateInfo.java | 88 ++++++++++++++++++++++ .../webapp/TestRMWebServicesCapacitySched.java | 6 +- 3 files changed, 106 insertions(+), 2 deletions(-) create mode 100644 hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/LeafQueueTemplateInfo.java diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/CapacitySchedulerQueueInfo.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/CapacitySchedulerQueueInfo.java index f4d4070..6e55b2f 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/CapacitySchedulerQueueInfo.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/CapacitySchedulerQueueInfo.java @@ -77,6 +77,8 @@ protected QueueAclsInfo queueAcls; protected int queuePriority; protected String orderingPolicyInfo; + protected boolean autoCreateChildQueueEnabled; + protected LeafQueueTemplateInfo leafQueueTemplate; CapacitySchedulerQueueInfo() { }; @@ -151,6 +153,10 @@ orderingPolicyInfo = ((ParentQueue) q).getQueueOrderingPolicy() .getConfigName(); } + + autoCreateChildQueueEnabled = conf. + isAutoCreateChildQueueEnabled(queuePath); + leafQueueTemplate = new LeafQueueTemplateInfo(conf, queuePath); } protected void populateQueueResourceUsage(ResourceUsage queueResourceUsage) { @@ -276,4 +282,12 @@ public String getOrderingPolicyInfo() { public boolean isLeafQueue() { return getQueues() == null; } + + public boolean isAutoCreateChildQueueEnabled() { + return autoCreateChildQueueEnabled; + } + + public LeafQueueTemplateInfo getLeafQueueTemplate() { + return leafQueueTemplate; + } } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/LeafQueueTemplateInfo.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/LeafQueueTemplateInfo.java new file mode 100644 index 0000000..9e099d1 --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/LeafQueueTemplateInfo.java @@ -0,0 +1,88 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.yarn.server.resourcemanager.webapp.dao; + +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacitySchedulerConfiguration; + +import javax.xml.bind.annotation.XmlAccessType; +import javax.xml.bind.annotation.XmlAccessorType; +import javax.xml.bind.annotation.XmlRootElement; +import java.util.ArrayList; + +import static org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacitySchedulerConfiguration.AUTO_CREATED_LEAF_QUEUE_TEMPLATE_PREFIX; +import static org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacitySchedulerConfiguration.DOT; + +/** + * This class stores the LeafQueue Template configuration. + */ +@XmlRootElement +@XmlAccessorType(XmlAccessType.FIELD) +public class LeafQueueTemplateInfo { + + private ArrayList property = new ArrayList<>(); + + public LeafQueueTemplateInfo() { + } // JAXB needs this + + public LeafQueueTemplateInfo(Configuration conf, String queuePath) { + String configPrefix = CapacitySchedulerConfiguration. + getQueuePrefix(queuePath) + AUTO_CREATED_LEAF_QUEUE_TEMPLATE_PREFIX + + DOT; + conf.forEach(entry -> { + if (entry.getKey().startsWith(configPrefix)) { + add(new ConfItem(entry.getKey(), entry.getValue())); + } + }); + } + + public void add(ConfItem confItem) { + property.add(confItem); + } + + public ArrayList getItems() { + return property; + } + + /** + * This class stores the Configuration Property. + */ + @XmlAccessorType(XmlAccessType.FIELD) + public static class ConfItem { + + private String name; + private String value; + + public ConfItem() { + // JAXB needs this + } + + public ConfItem(String name, String value){ + this.name = name; + this.value = value; + } + + public String getKey() { + return name; + } + + public String getValue() { + return value; + } + } +} diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesCapacitySched.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesCapacitySched.java index 324a392..5cc1332 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesCapacitySched.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesCapacitySched.java @@ -19,6 +19,7 @@ package org.apache.hadoop.yarn.server.resourcemanager.webapp; import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; import static org.junit.Assert.assertNotNull; import static org.junit.Assert.assertTrue; import static org.junit.Assert.fail; @@ -384,10 +385,10 @@ private void verifyClusterSchedulerGeneric(String type, float usedCapacity, private void verifySubQueue(JSONObject info, String q, float parentAbsCapacity, float parentAbsMaxCapacity) throws JSONException, Exception { - int numExpectedElements = 24; + int numExpectedElements = 26; boolean isParentQueue = true; if (!info.has("queues")) { - numExpectedElements = 42; + numExpectedElements = 44; isParentQueue = false; } assertEquals("incorrect number of elements", numExpectedElements, info.length()); @@ -423,6 +424,7 @@ private void verifySubQueue(JSONObject info, String q, assertEquals("0", info.getString("queuePriority")); assertEquals("utilization", info.getString("orderingPolicyInfo")); + assertFalse(info.getBoolean("autoCreateChildQueueEnabled")); } else { Assert.assertEquals("\"type\" field is incorrect", "capacitySchedulerLeafQueueInfo", info.getString("type")); -- 2.7.4 (Apple Git-66)