diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacitySchedulerConfiguration.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacitySchedulerConfiguration.java index 547d1319100..5098c92b96e 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacitySchedulerConfiguration.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacitySchedulerConfiguration.java @@ -471,6 +471,16 @@ public void setCapacity(String queue, float capacity) { ", capacity=" + capacity); } + public void setCapacity(String queue, String absoluteResourceCapacity) { + if (queue.equals("root")) { + throw new IllegalArgumentException( + "Cannot set capacity, root queue has a fixed capacity"); + } + set(getQueuePrefix(queue) + CAPACITY, absoluteResourceCapacity); + LOG.debug("CSConf - setCapacity: queuePrefix=" + getQueuePrefix(queue) + + ", capacity=" + absoluteResourceCapacity); + } + public float getNonLabeledQueueMaximumCapacity(String queue) { String configuredCapacity = get(getQueuePrefix(queue) + MAXIMUM_CAPACITY); boolean matcher = (configuredCapacity != null) @@ -505,7 +515,11 @@ public void setMaximumCapacity(String queue, float maxCapacity) { public void setCapacityByLabel(String queue, String label, float capacity) { setFloat(getNodeLabelPrefix(queue, label) + CAPACITY, capacity); } - + + public void setCapacityByLabel(String queue, String label, String absoluteResourceCapacity) { + set(getNodeLabelPrefix(queue, label) + CAPACITY, absoluteResourceCapacity); + } + public void setMaximumCapacityByLabel(String queue, String label, float capacity) { setFloat(getNodeLabelPrefix(queue, label) + MAXIMUM_CAPACITY, capacity); @@ -642,8 +656,9 @@ public void setAccessibleNodeLabels(String queue, Set labels) { private float internalGetLabeledQueueCapacity(String queue, String label, String suffix, float defaultValue) { String capacityPropertyName = getNodeLabelPrefix(queue, label) + suffix; - boolean matcher = (capacityPropertyName != null) - && RESOURCE_PATTERN.matcher(capacityPropertyName).find(); + String configuredCapacity = get(capacityPropertyName); + boolean matcher = (configuredCapacity != null) + && RESOURCE_PATTERN.matcher(configuredCapacity).find(); if (matcher) { // Return capacity in percentage as 0 for non-root queues and 100 for // root.From AbstractCSQueue, absolute resource will be parsed and diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacityScheduler.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacityScheduler.java index db8836aa87f..5958f0ccacf 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacityScheduler.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacityScheduler.java @@ -40,6 +40,7 @@ import java.util.concurrent.BrokenBarrierException; import java.util.concurrent.CyclicBarrier; +import com.google.common.collect.Sets; import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; import org.apache.hadoop.conf.Configuration; @@ -123,17 +124,7 @@ import org.apache.hadoop.yarn.server.resourcemanager.rmcontainer.RMContainerState; import org.apache.hadoop.yarn.server.resourcemanager.rmnode.RMNode; import org.apache.hadoop.yarn.server.resourcemanager.rmnode.RMNodeResourceUpdateEvent; -import org.apache.hadoop.yarn.server.resourcemanager.scheduler.AbstractYarnScheduler; -import org.apache.hadoop.yarn.server.resourcemanager.scheduler.Allocation; -import org.apache.hadoop.yarn.server.resourcemanager.scheduler.ContainerUpdates; -import org.apache.hadoop.yarn.server.resourcemanager.scheduler.QueueMetrics; -import org.apache.hadoop.yarn.server.resourcemanager.scheduler.ResourceScheduler; -import org.apache.hadoop.yarn.server.resourcemanager.scheduler.SchedulerApplication; -import org.apache.hadoop.yarn.server.resourcemanager.scheduler.SchedulerApplicationAttempt; -import org.apache.hadoop.yarn.server.resourcemanager.scheduler.SchedulerNode; -import org.apache.hadoop.yarn.server.resourcemanager.scheduler.SchedulerNodeReport; -import org.apache.hadoop.yarn.server.resourcemanager.scheduler.TestSchedulerUtils; -import org.apache.hadoop.yarn.server.resourcemanager.scheduler.YarnScheduler; +import org.apache.hadoop.yarn.server.resourcemanager.scheduler.*; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.allocator.AllocationState; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.allocator.ContainerAllocation; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.common.ResourceCommitRequest; @@ -172,7 +163,6 @@ import com.google.common.base.Supplier; import com.google.common.collect.ImmutableMap; import com.google.common.collect.ImmutableSet; -import com.google.common.collect.Sets; import org.mockito.invocation.InvocationOnMock; import org.mockito.stubbing.Answer; @@ -957,6 +947,41 @@ public void testParseQueue() throws IOException { new ClientToAMTokenSecretManagerInRM(), null)); } + @Test + public void testParseQueueWithAbsoluteResource() { + String childQueue = "testQueue"; + String labelName = "testLabel"; + + CapacityScheduler cs = new CapacityScheduler(); + cs.setConf(new YarnConfiguration()); + cs.setRMContext(resourceManager.getRMContext()); + CapacitySchedulerConfiguration conf = new CapacitySchedulerConfiguration(); + + conf.setQueues("root", new String[]{childQueue}); + conf.setCapacity("root." + childQueue, "[memory=20480,vcores=200]"); + conf.setAccessibleNodeLabels("root." + childQueue, Sets.newHashSet(labelName)); + conf.setCapacityByLabel("root", labelName, "[memory=10240,vcores=100]"); + conf.setCapacityByLabel("root." + childQueue, labelName, "[memory=4096,vcores=10]"); + + cs.init(conf); + cs.start();/ + + Resource rootQueueLableCapacity = cs.getQueue("root") + .getQueueResourceQuotas().getConfiguredMinResource(labelName); + assertEquals(10240, rootQueueLableCapacity.getMemorySize()); + assertEquals(100, rootQueueLableCapacity.getVirtualCores()); + + QueueResourceQuotas childQueueQuotas = cs.getQueue(childQueue).getQueueResourceQuotas(); + Resource childQueueCapacity = childQueueQuotas.getConfiguredMinResource(); + assertEquals(20480, childQueueCapacity.getMemorySize()); + assertEquals(200, childQueueCapacity.getVirtualCores()); + + Resource childQueueLabelCapacity = childQueueQuotas.getConfiguredMinResource(labelName); + assertEquals(4096, childQueueLabelCapacity.getMemorySize()); + assertEquals(10, childQueueLabelCapacity.getVirtualCores()); + + } + @Test public void testReconnectedNode() throws Exception { CapacitySchedulerConfiguration csConf =