From ec602d7f769ed75e4aa616b00177fd218a4f7486 Mon Sep 17 00:00:00 2001 From: Prabhu Joseph Date: Fri, 17 Apr 2020 11:17:32 +0530 Subject: [PATCH] YARN-10237. Add isAbsoluteResource config for queue in scheduler response Signed-off-by: Prabhu Joseph --- .../capacity/CapacitySchedulerConfiguration.java | 3 +- .../webapp/dao/CapacitySchedulerQueueInfo.java | 15 ++++ .../webapp/dao/LeafQueueTemplateInfo.java | 91 ++++++++++++++++++++++ .../webapp/TestRMWebServicesCapacitySched.java | 19 ++++- 4 files changed, 125 insertions(+), 3 deletions(-) create mode 100644 hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/LeafQueueTemplateInfo.java diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacitySchedulerConfiguration.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacitySchedulerConfiguration.java index 3ae66ca..7edf2da 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacitySchedulerConfiguration.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacitySchedulerConfiguration.java @@ -377,7 +377,8 @@ public static final String PATTERN_FOR_ABSOLUTE_RESOURCE = "^\\[[\\w\\.,\\-_=\\ /]+\\]$"; - private static final Pattern RESOURCE_PATTERN = Pattern.compile(PATTERN_FOR_ABSOLUTE_RESOURCE); + public static final Pattern RESOURCE_PATTERN = + Pattern.compile(PATTERN_FOR_ABSOLUTE_RESOURCE); /** * Different resource types supported. diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/CapacitySchedulerQueueInfo.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/CapacitySchedulerQueueInfo.java index f4d4070..947c44c 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/CapacitySchedulerQueueInfo.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/CapacitySchedulerQueueInfo.java @@ -42,6 +42,11 @@ import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.PlanQueue; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.QueueCapacities; +import static org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity. + CapacitySchedulerConfiguration.RESOURCE_PATTERN; +import static org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity. + CapacitySchedulerConfiguration.CAPACITY; + @XmlRootElement @XmlAccessorType(XmlAccessType.FIELD) @XmlSeeAlso({CapacitySchedulerLeafQueueInfo.class}) @@ -61,6 +66,7 @@ protected float absoluteUsedCapacity; protected int numApplications; protected String queueName; + protected boolean isAbsoluteResource; protected QueueState state; protected CapacitySchedulerQueueInfoList queues; protected ResourceInfo resourcesUsed; @@ -151,6 +157,11 @@ orderingPolicyInfo = ((ParentQueue) q).getQueueOrderingPolicy() .getConfigName(); } + + String configuredCapacity = conf.get( + CapacitySchedulerConfiguration.getQueuePrefix(queuePath) + CAPACITY); + isAbsoluteResource = (configuredCapacity != null) + && RESOURCE_PATTERN.matcher(configuredCapacity).find(); } protected void populateQueueResourceUsage(ResourceUsage queueResourceUsage) { @@ -203,6 +214,10 @@ public long getPendingContainers() { return pendingContainers; } + public boolean isAbsoluteResource() { + return isAbsoluteResource; + } + public String getQueueName() { return this.queueName; } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/LeafQueueTemplateInfo.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/LeafQueueTemplateInfo.java new file mode 100644 index 0000000..f528a47 --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/LeafQueueTemplateInfo.java @@ -0,0 +1,91 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.yarn.server.resourcemanager.webapp.dao; + +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacitySchedulerConfiguration; + +import javax.xml.bind.annotation.XmlAccessType; +import javax.xml.bind.annotation.XmlAccessorType; +import javax.xml.bind.annotation.XmlRootElement; +import java.util.ArrayList; + +import static org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacitySchedulerConfiguration.AUTO_CREATED_LEAF_QUEUE_TEMPLATE_PREFIX; +import static org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacitySchedulerConfiguration.DOT; + +/** + * This class stores the LeafQueue Template configuration. + */ +@XmlRootElement +@XmlAccessorType(XmlAccessType.FIELD) +public class LeafQueueTemplateInfo { + + private ArrayList property = new ArrayList<>(); + + public LeafQueueTemplateInfo() { + } // JAXB needs this + + public LeafQueueTemplateInfo(Configuration conf, String queuePath) { + String configPrefix = CapacitySchedulerConfiguration. + getQueuePrefix(queuePath) + AUTO_CREATED_LEAF_QUEUE_TEMPLATE_PREFIX + + DOT; + conf.forEach(entry -> { + if (entry.getKey().startsWith(configPrefix)) { + String name = entry.getKey(); + int start = name.lastIndexOf(AUTO_CREATED_LEAF_QUEUE_TEMPLATE_PREFIX + + DOT); + add(new ConfItem(name.substring(start), entry.getValue())); + } + }); + } + + public void add(ConfItem confItem) { + property.add(confItem); + } + + public ArrayList getItems() { + return property; + } + + /** + * This class stores the Configuration Property. + */ + @XmlAccessorType(XmlAccessType.FIELD) + public static class ConfItem { + + private String name; + private String value; + + public ConfItem() { + // JAXB needs this + } + + public ConfItem(String name, String value){ + this.name = name; + this.value = value; + } + + public String getKey() { + return name; + } + + public String getValue() { + return value; + } + } +} diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesCapacitySched.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesCapacitySched.java index 324a392..8098622 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesCapacitySched.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesCapacitySched.java @@ -20,6 +20,7 @@ import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertNotNull; +import static org.junit.Assert.assertFalse; import static org.junit.Assert.assertTrue; import static org.junit.Assert.fail; @@ -79,6 +80,7 @@ int numApplications; String queueName; String state; + boolean isAbsoluteResource; } private class LeafQueueInfo extends QueueInfo { @@ -131,6 +133,9 @@ private static void setupQueueConfiguration( final String B = CapacitySchedulerConfiguration.ROOT + ".b"; config.setCapacity(B, 89.5f); + final String C = CapacitySchedulerConfiguration.ROOT + ".c"; + config.setCapacity(C, "[memory=1024]"); + // Define 2nd-level queues final String A1 = A + ".a1"; final String A2 = A + ".a2"; @@ -290,6 +295,8 @@ public void verifySubQueueXML(Element qElem, String q, WebServicesTestUtils.getXmlInt(qElem, "numApplications"); qi.queueName = WebServicesTestUtils.getXmlString(qElem, "queueName"); qi.state = WebServicesTestUtils.getXmlString(qElem, "state"); + qi.isAbsoluteResource = WebServicesTestUtils.getXmlBoolean(qElem, + "isAbsoluteResource"); verifySubQueueGeneric(q, qi, parentAbsCapacity, parentAbsMaxCapacity); if (hasSubQueues) { for (int j = 0; j < children.getLength(); j++) { @@ -384,10 +391,10 @@ private void verifyClusterSchedulerGeneric(String type, float usedCapacity, private void verifySubQueue(JSONObject info, String q, float parentAbsCapacity, float parentAbsMaxCapacity) throws JSONException, Exception { - int numExpectedElements = 24; + int numExpectedElements = 25; boolean isParentQueue = true; if (!info.has("queues")) { - numExpectedElements = 42; + numExpectedElements = 43; isParentQueue = false; } assertEquals("incorrect number of elements", numExpectedElements, info.length()); @@ -471,6 +478,14 @@ private void verifySubQueueGeneric(String q, QueueInfo info, + " expected: " + q, qshortName.matches(info.queueName)); assertTrue("state doesn't match", (csConf.getState(q).toString()).matches(info.state)); + if (q.equals("c")) { + assertTrue("c queue is not configured in Absolute resource", + info.isAbsoluteResource); + } else { + assertFalse(info.queueName + + " queue is not configured in Absolute resource", + info.isAbsoluteResource); + } } -- 2.7.4 (Apple Git-66)