From a44eed66ba685f2d7489c5dbc813244c0a39dcd3 Mon Sep 17 00:00:00 2001 From: Prabhu Joseph Date: Fri, 17 Apr 2020 11:17:32 +0530 Subject: [PATCH] YARN-10237. Add isAbsoluteResource config for queue in scheduler response Signed-off-by: Prabhu Joseph --- .../capacity/CapacitySchedulerConfiguration.java | 2 +- .../webapp/dao/CapacitySchedulerQueueInfo.java | 16 ++++++++++++++++ .../webapp/TestRMWebServicesCapacitySched.java | 19 +++++++++++++++++-- 3 files changed, 34 insertions(+), 3 deletions(-) diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacitySchedulerConfiguration.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacitySchedulerConfiguration.java index c262fec..f060b56 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacitySchedulerConfiguration.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacitySchedulerConfiguration.java @@ -377,7 +377,7 @@ public static final String PATTERN_FOR_ABSOLUTE_RESOURCE = "^\\[[\\w\\.,\\-_=\\ /]+\\]$"; - private static final Pattern RESOURCE_PATTERN = Pattern.compile(PATTERN_FOR_ABSOLUTE_RESOURCE); + public static final Pattern RESOURCE_PATTERN = Pattern.compile(PATTERN_FOR_ABSOLUTE_RESOURCE); /** * Different resource types supported. diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/CapacitySchedulerQueueInfo.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/CapacitySchedulerQueueInfo.java index f4d4070..2231e51 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/CapacitySchedulerQueueInfo.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/CapacitySchedulerQueueInfo.java @@ -22,6 +22,7 @@ import java.util.HashMap; import java.util.Map; import java.util.Set; +import java.util.regex.Pattern; import javax.xml.bind.annotation.XmlAccessType; import javax.xml.bind.annotation.XmlAccessorType; @@ -42,6 +43,11 @@ import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.PlanQueue; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.QueueCapacities; +import static org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity. + CapacitySchedulerConfiguration.RESOURCE_PATTERN; +import static org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity. + CapacitySchedulerConfiguration.CAPACITY; + @XmlRootElement @XmlAccessorType(XmlAccessType.FIELD) @XmlSeeAlso({CapacitySchedulerLeafQueueInfo.class}) @@ -61,6 +67,7 @@ protected float absoluteUsedCapacity; protected int numApplications; protected String queueName; + protected boolean isAbsoluteResource; protected QueueState state; protected CapacitySchedulerQueueInfoList queues; protected ResourceInfo resourcesUsed; @@ -151,6 +158,11 @@ orderingPolicyInfo = ((ParentQueue) q).getQueueOrderingPolicy() .getConfigName(); } + + String configuredCapacity = conf.get( + CapacitySchedulerConfiguration.getQueuePrefix(queuePath) + CAPACITY); + isAbsoluteResource = (configuredCapacity != null) + && RESOURCE_PATTERN.matcher(configuredCapacity).find(); } protected void populateQueueResourceUsage(ResourceUsage queueResourceUsage) { @@ -203,6 +215,10 @@ public long getPendingContainers() { return pendingContainers; } + public boolean isAbsoluteResource() { + return isAbsoluteResource; + } + public String getQueueName() { return this.queueName; } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesCapacitySched.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesCapacitySched.java index 324a392..8098622 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesCapacitySched.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesCapacitySched.java @@ -20,6 +20,7 @@ import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertNotNull; +import static org.junit.Assert.assertFalse; import static org.junit.Assert.assertTrue; import static org.junit.Assert.fail; @@ -79,6 +80,7 @@ int numApplications; String queueName; String state; + boolean isAbsoluteResource; } private class LeafQueueInfo extends QueueInfo { @@ -131,6 +133,9 @@ private static void setupQueueConfiguration( final String B = CapacitySchedulerConfiguration.ROOT + ".b"; config.setCapacity(B, 89.5f); + final String C = CapacitySchedulerConfiguration.ROOT + ".c"; + config.setCapacity(C, "[memory=1024]"); + // Define 2nd-level queues final String A1 = A + ".a1"; final String A2 = A + ".a2"; @@ -290,6 +295,8 @@ public void verifySubQueueXML(Element qElem, String q, WebServicesTestUtils.getXmlInt(qElem, "numApplications"); qi.queueName = WebServicesTestUtils.getXmlString(qElem, "queueName"); qi.state = WebServicesTestUtils.getXmlString(qElem, "state"); + qi.isAbsoluteResource = WebServicesTestUtils.getXmlBoolean(qElem, + "isAbsoluteResource"); verifySubQueueGeneric(q, qi, parentAbsCapacity, parentAbsMaxCapacity); if (hasSubQueues) { for (int j = 0; j < children.getLength(); j++) { @@ -384,10 +391,10 @@ private void verifyClusterSchedulerGeneric(String type, float usedCapacity, private void verifySubQueue(JSONObject info, String q, float parentAbsCapacity, float parentAbsMaxCapacity) throws JSONException, Exception { - int numExpectedElements = 24; + int numExpectedElements = 25; boolean isParentQueue = true; if (!info.has("queues")) { - numExpectedElements = 42; + numExpectedElements = 43; isParentQueue = false; } assertEquals("incorrect number of elements", numExpectedElements, info.length()); @@ -471,6 +478,14 @@ private void verifySubQueueGeneric(String q, QueueInfo info, + " expected: " + q, qshortName.matches(info.queueName)); assertTrue("state doesn't match", (csConf.getState(q).toString()).matches(info.state)); + if (q.equals("c")) { + assertTrue("c queue is not configured in Absolute resource", + info.isAbsoluteResource); + } else { + assertFalse(info.queueName + + " queue is not configured in Absolute resource", + info.isAbsoluteResource); + } } -- 2.7.4 (Apple Git-66)