From 7623a31cda7059f3e9175346b1ccf21053ed5622 Mon Sep 17 00:00:00 2001 From: Prabhu Joseph Date: Fri, 17 Apr 2020 11:17:32 +0530 Subject: [PATCH] YARN-10237. Add isAbsoluteResource config for queue in scheduler response Signed-off-by: Prabhu Joseph --- .../webapp/dao/CapacitySchedulerQueueInfo.java | 19 +++++++++++++++++++ .../webapp/TestRMWebServicesCapacitySched.java | 19 +++++++++++++++++-- 2 files changed, 36 insertions(+), 2 deletions(-) diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/CapacitySchedulerQueueInfo.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/CapacitySchedulerQueueInfo.java index f4d4070..091d367 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/CapacitySchedulerQueueInfo.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/CapacitySchedulerQueueInfo.java @@ -22,6 +22,7 @@ import java.util.HashMap; import java.util.Map; import java.util.Set; +import java.util.regex.Pattern; import javax.xml.bind.annotation.XmlAccessType; import javax.xml.bind.annotation.XmlAccessorType; @@ -42,6 +43,11 @@ import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.PlanQueue; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.QueueCapacities; +import static org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity. + CapacitySchedulerConfiguration.PATTERN_FOR_ABSOLUTE_RESOURCE; +import static org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity. + CapacitySchedulerConfiguration.CAPACITY; + @XmlRootElement @XmlAccessorType(XmlAccessType.FIELD) @XmlSeeAlso({CapacitySchedulerLeafQueueInfo.class}) @@ -61,6 +67,7 @@ protected float absoluteUsedCapacity; protected int numApplications; protected String queueName; + protected boolean isAbsoluteResource; protected QueueState state; protected CapacitySchedulerQueueInfoList queues; protected ResourceInfo resourcesUsed; @@ -151,8 +158,16 @@ orderingPolicyInfo = ((ParentQueue) q).getQueueOrderingPolicy() .getConfigName(); } + + Pattern resourcePattern = Pattern.compile(PATTERN_FOR_ABSOLUTE_RESOURCE); + String configuredCapacity = conf.get( + CapacitySchedulerConfiguration.getQueuePrefix(queuePath) + CAPACITY); + isAbsoluteResource = (configuredCapacity != null) + && resourcePattern.matcher(configuredCapacity).find(); } + + protected void populateQueueResourceUsage(ResourceUsage queueResourceUsage) { resources = new ResourcesInfo(queueResourceUsage, false); } @@ -203,6 +218,10 @@ public long getPendingContainers() { return pendingContainers; } + public boolean isAbsoluteResource() { + return isAbsoluteResource; + } + public String getQueueName() { return this.queueName; } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesCapacitySched.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesCapacitySched.java index 324a392..8098622 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesCapacitySched.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesCapacitySched.java @@ -20,6 +20,7 @@ import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertNotNull; +import static org.junit.Assert.assertFalse; import static org.junit.Assert.assertTrue; import static org.junit.Assert.fail; @@ -79,6 +80,7 @@ int numApplications; String queueName; String state; + boolean isAbsoluteResource; } private class LeafQueueInfo extends QueueInfo { @@ -131,6 +133,9 @@ private static void setupQueueConfiguration( final String B = CapacitySchedulerConfiguration.ROOT + ".b"; config.setCapacity(B, 89.5f); + final String C = CapacitySchedulerConfiguration.ROOT + ".c"; + config.setCapacity(C, "[memory=1024]"); + // Define 2nd-level queues final String A1 = A + ".a1"; final String A2 = A + ".a2"; @@ -290,6 +295,8 @@ public void verifySubQueueXML(Element qElem, String q, WebServicesTestUtils.getXmlInt(qElem, "numApplications"); qi.queueName = WebServicesTestUtils.getXmlString(qElem, "queueName"); qi.state = WebServicesTestUtils.getXmlString(qElem, "state"); + qi.isAbsoluteResource = WebServicesTestUtils.getXmlBoolean(qElem, + "isAbsoluteResource"); verifySubQueueGeneric(q, qi, parentAbsCapacity, parentAbsMaxCapacity); if (hasSubQueues) { for (int j = 0; j < children.getLength(); j++) { @@ -384,10 +391,10 @@ private void verifyClusterSchedulerGeneric(String type, float usedCapacity, private void verifySubQueue(JSONObject info, String q, float parentAbsCapacity, float parentAbsMaxCapacity) throws JSONException, Exception { - int numExpectedElements = 24; + int numExpectedElements = 25; boolean isParentQueue = true; if (!info.has("queues")) { - numExpectedElements = 42; + numExpectedElements = 43; isParentQueue = false; } assertEquals("incorrect number of elements", numExpectedElements, info.length()); @@ -471,6 +478,14 @@ private void verifySubQueueGeneric(String q, QueueInfo info, + " expected: " + q, qshortName.matches(info.queueName)); assertTrue("state doesn't match", (csConf.getState(q).toString()).matches(info.state)); + if (q.equals("c")) { + assertTrue("c queue is not configured in Absolute resource", + info.isAbsoluteResource); + } else { + assertFalse(info.queueName + + " queue is not configured in Absolute resource", + info.isAbsoluteResource); + } } -- 2.7.4 (Apple Git-66)