diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/AllocationConfiguration.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/AllocationConfiguration.java index f143aa6..7f6bd78 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/AllocationConfiguration.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/AllocationConfiguration.java @@ -50,9 +50,9 @@ private final Map minQueueResources; // Maximum amount of resources per queue @VisibleForTesting - final Map maxQueueResources; + final Map maxQueueResources; // Maximum amount of resources for each queue's ad hoc children - private final Map maxChildQueueResources; + private final Map maxChildQueueResources; // Sharing weights for each queue private final Map queueWeights; @@ -64,7 +64,7 @@ final Map userMaxApps; private final int userMaxAppsDefault; private final int queueMaxAppsDefault; - private final Resource queueMaxResourcesDefault; + private final ResourceConfiguration queueMaxResourcesDefault; // Maximum resource share for each leaf queue that can be used to run AMs final Map queueMaxAMShares; @@ -113,12 +113,12 @@ private final Set nonPreemptableQueues; public AllocationConfiguration(Map minQueueResources, - Map maxQueueResources, - Map maxChildQueueResources, + Map maxQueueResources, + Map maxChildQueueResources, Map queueMaxApps, Map userMaxApps, Map queueWeights, Map queueMaxAMShares, int userMaxAppsDefault, - int queueMaxAppsDefault, Resource queueMaxResourcesDefault, + int queueMaxAppsDefault, ResourceConfiguration queueMaxResourcesDefault, float queueMaxAMShareDefault, Map schedulingPolicies, SchedulingPolicy defaultSchedulingPolicy, @@ -167,7 +167,7 @@ public AllocationConfiguration(Configuration conf) { queueMaxAMShares = new HashMap<>(); userMaxAppsDefault = Integer.MAX_VALUE; queueMaxAppsDefault = Integer.MAX_VALUE; - queueMaxResourcesDefault = Resources.unbounded(); + queueMaxResourcesDefault = new ResourceConfiguration(Resources.unbounded()); queueMaxAMShareDefault = 0.5f; queueAcls = new HashMap<>(); resAcls = new HashMap<>(); @@ -293,26 +293,15 @@ Resource getMinResources(String queue) { /** * Get the maximum resource allocation for the given queue. If the max in not - * set, return the larger of the min and the default max. + * set, return the default max. * * @param queue the target queue's name * @return the max allocation on this queue */ - @VisibleForTesting - Resource getMaxResources(String queue) { - Resource maxQueueResource = maxQueueResources.get(queue); - if (maxQueueResource == null) { - Resource minQueueResource = minQueueResources.get(queue); - if (minQueueResource != null && - Resources.greaterThan(RESOURCE_CALCULATOR, Resources.unbounded(), - minQueueResource, queueMaxResourcesDefault)) { - return minQueueResource; - } else { - return queueMaxResourcesDefault; - } - } else { - return maxQueueResource; - } + @VisibleForTesting ResourceConfiguration getMaxResources(String queue) { + ResourceConfiguration maxQueueResource = maxQueueResources.get(queue); + return maxQueueResource == null ? + queueMaxResourcesDefault : maxQueueResource; } /** @@ -321,8 +310,7 @@ Resource getMaxResources(String queue) { * @param queue the target queue's name * @return the max allocation on this queue or null if not set */ - @VisibleForTesting - Resource getMaxChildResources(String queue) { + @VisibleForTesting ResourceConfiguration getMaxChildResources(String queue) { return maxChildQueueResources.get(queue); } @@ -421,7 +409,7 @@ public void initFSQueue(FSQueue queue){ // Set queue metrics. queue.getMetrics().setMinShare(getMinResources(name)); - queue.getMetrics().setMaxShare(getMaxResources(name)); + queue.getMetrics().setMaxShare(queue.getMaxShare()); queue.getMetrics().setMaxApps(getQueueMaxApps(name)); queue.getMetrics().setSchedulingPolicy(getSchedulingPolicy(name).getName()); } diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/AllocationFileLoaderService.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/AllocationFileLoaderService.java index bc204cb..b891e8a 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/AllocationFileLoaderService.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/AllocationFileLoaderService.java @@ -227,8 +227,8 @@ public synchronized void reloadAllocations() throws IOException, // Create some temporary hashmaps to hold the new allocs, and we only save // them in our fields if we have parsed the entire allocs file successfully. Map minQueueResources = new HashMap<>(); - Map maxQueueResources = new HashMap<>(); - Map maxChildQueueResources = new HashMap<>(); + Map maxQueueResources = new HashMap<>(); + Map maxChildQueueResources = new HashMap<>(); Map queueMaxApps = new HashMap<>(); Map userMaxApps = new HashMap<>(); Map queueMaxAMShares = new HashMap<>(); @@ -245,7 +245,8 @@ public synchronized void reloadAllocations() throws IOException, Set nonPreemptableQueues = new HashSet<>(); int userMaxAppsDefault = Integer.MAX_VALUE; int queueMaxAppsDefault = Integer.MAX_VALUE; - Resource queueMaxResourcesDefault = Resources.unbounded(); + ResourceConfiguration queueMaxResourcesDefault = + new ResourceConfiguration(Resources.unbounded()); float queueMaxAMShareDefault = 0.5f; long defaultFairSharePreemptionTimeout = Long.MAX_VALUE; long defaultMinSharePreemptionTimeout = Long.MAX_VALUE; @@ -305,7 +306,7 @@ public synchronized void reloadAllocations() throws IOException, } } else if ("queueMaxResourcesDefault".equals(element.getTagName())) { String text = ((Text)element.getFirstChild()).getData().trim(); - Resource val = + ResourceConfiguration val = FairSchedulerConfiguration.parseResourceConfigValue(text); queueMaxResourcesDefault = val; } else if ("userMaxAppsDefault".equals(element.getTagName())) { @@ -448,8 +449,8 @@ public synchronized void reloadAllocations() throws IOException, */ private void loadQueue(String parentName, Element element, Map minQueueResources, - Map maxQueueResources, - Map maxChildQueueResources, + Map maxQueueResources, + Map maxChildQueueResources, Map queueMaxApps, Map userMaxApps, Map queueMaxAMShares, @@ -495,17 +496,17 @@ private void loadQueue(String parentName, Element element, Element field = (Element) fieldNode; if ("minResources".equals(field.getTagName())) { String text = ((Text)field.getFirstChild()).getData().trim(); - Resource val = + ResourceConfiguration val = FairSchedulerConfiguration.parseResourceConfigValue(text); - minQueueResources.put(queueName, val); + minQueueResources.put(queueName, val.getResource()); } else if ("maxResources".equals(field.getTagName())) { String text = ((Text)field.getFirstChild()).getData().trim(); - Resource val = + ResourceConfiguration val = FairSchedulerConfiguration.parseResourceConfigValue(text); maxQueueResources.put(queueName, val); } else if ("maxChildResources".equals(field.getTagName())) { String text = ((Text)field.getFirstChild()).getData().trim(); - Resource val = + ResourceConfiguration val = FairSchedulerConfiguration.parseResourceConfigValue(text); maxChildQueueResources.put(queueName, val); } else if ("maxRunningApps".equals(field.getTagName())) { @@ -605,13 +606,15 @@ private void loadQueue(String parentName, Element element, queueAcls.put(queueName, acls); resAcls.put(queueName, racls); - if (maxQueueResources.containsKey(queueName) && - minQueueResources.containsKey(queueName) - && !Resources.fitsIn(minQueueResources.get(queueName), - maxQueueResources.get(queueName))) { + ResourceConfiguration maxResourceConf = maxQueueResources.get(queueName); + if (maxResourceConf != null && + !maxResourceConf.isPercentage() && + minQueueResources.containsKey(queueName) && + !Resources.fitsIn(minQueueResources.get(queueName), + maxResourceConf.getResource())) { LOG.warn( String.format("Queue %s has max resources %s less than " - + "min resources %s", queueName, maxQueueResources.get(queueName), + + "min resources %s", queueName, maxResourceConf.getResource(), minQueueResources.get(queueName))); } } diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSLeafQueue.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSLeafQueue.java index b911a1a..301147c 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSLeafQueue.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSLeafQueue.java @@ -343,10 +343,10 @@ public void updateDemand() { readLock.unlock(); } // Cap demand to maxShare to limit allocation to maxShare - demand = Resources.componentwiseMin(tmpDemand, maxShare); + demand = Resources.componentwiseMin(tmpDemand, getMaxShare()); if (LOG.isDebugEnabled()) { LOG.debug("The updated demand for " + getName() + " is " + demand - + "; the max is " + maxShare); + + "; the max is " + getMaxShare()); LOG.debug("The updated fairshare for " + getName() + " is " + getFairShare()); } @@ -622,7 +622,7 @@ protected void dumpStateInternal(StringBuilder sb) { ", Policy: " + policy.getName() + ", FairShare: " + getFairShare() + ", SteadyFairShare: " + getSteadyFairShare() + - ", MaxShare: " + maxShare + + ", MaxShare: " + getMaxShare() + ", MinShare: " + minShare + ", ResourceUsage: " + getResourceUsage() + ", Demand: " + getDemand() + diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSParentQueue.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSParentQueue.java index 5b4e4dc..e42c9f5 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSParentQueue.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSParentQueue.java @@ -150,13 +150,13 @@ public void updateDemand() { } } // Cap demand to maxShare to limit allocation to maxShare - demand = Resources.componentwiseMin(demand, maxShare); + demand = Resources.componentwiseMin(demand, getMaxShare()); } finally { writeLock.unlock(); } if (LOG.isDebugEnabled()) { LOG.debug("The updated demand for " + getName() + " is " + demand + - "; the max is " + maxShare); + "; the max is " + getMaxShare()); } } @@ -300,7 +300,7 @@ protected void dumpStateInternal(StringBuilder sb) { ", Policy: " + policy.getName() + ", FairShare: " + getFairShare() + ", SteadyFairShare: " + getSteadyFairShare() + - ", MaxShare: " + maxShare + + ", MaxShare: " + getMaxShare() + ", MinShare: " + minShare + ", ResourceUsage: " + getResourceUsage() + ", Demand: " + getDemand() + diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSQueue.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSQueue.java index 1016823..1c5c8f2 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSQueue.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSQueue.java @@ -72,9 +72,9 @@ protected ResourceWeights weights; protected Resource minShare; - protected Resource maxShare; + private ResourceConfiguration maxShare; protected int maxRunningApps; - protected Resource maxChildQueueResource; + private ResourceConfiguration maxChildQueueResource; // maxAMShare is a value between 0 and 1. protected float maxAMShare; @@ -106,7 +106,7 @@ public FSQueue(String name, FairScheduler scheduler, FSParentQueue parent) { * * @param recursive whether child queues should be reinitialized recursively */ - public void reinit(boolean recursive) { + public final void reinit(boolean recursive) { AllocationConfiguration allocConf = scheduler.getAllocationConfiguration(); allocConf.initFSQueue(this); updatePreemptionVariables(); @@ -158,26 +158,33 @@ public Resource getMinShare() { return minShare; } - public void setMaxShare(Resource maxShare){ + public void setMaxShare(ResourceConfiguration maxShare){ this.maxShare = maxShare; } + @Override + public Resource getMaxShare() { + Resource maxResource = maxShare.getResource(scheduler.getClusterResource()); + // Set max resource to min resource if min resource is greater than max + // resource + if(Resources.greaterThan(scheduler.getResourceCalculator(), + scheduler.getClusterResource(), minShare, maxResource)) { + maxResource = minShare; + } + return maxResource; + } + public Resource getReservedResource() { reservedResource.setMemorySize(metrics.getReservedMB()); reservedResource.setVirtualCores(metrics.getReservedVirtualCores()); return reservedResource; } - @Override - public Resource getMaxShare() { - return maxShare; - } - - public void setMaxChildQueueResource(Resource maxChildShare){ + public void setMaxChildQueueResource(ResourceConfiguration maxChildShare){ this.maxChildQueueResource = maxChildShare; } - public Resource getMaxChildQueueResource() { + public ResourceConfiguration getMaxChildQueueResource() { return maxChildQueueResource; } @@ -416,7 +423,7 @@ boolean assignContainerPreCheck(FSSchedulerNode node) { + " because it has reserved containers."); } return false; - } else if (!Resources.fitsIn(getResourceUsage(), maxShare)) { + } else if (!Resources.fitsIn(getResourceUsage(), getMaxShare())) { if (LOG.isDebugEnabled()) { LOG.debug("Assigning container failed on node '" + node.getNodeName() + " because queue resource usage is larger than MaxShare: " diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FairScheduler.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FairScheduler.java index b41d3f7..8ae419b 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FairScheduler.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FairScheduler.java @@ -1549,7 +1549,8 @@ private void applyChildDefaults() { if ((queue.getParent() != null) && !configuredLeafQueues.contains(queue.getName()) && !configuredParentQueues.contains(queue.getName())) { - Resource max = queue.getParent().getMaxChildQueueResource(); + ResourceConfiguration max = queue.getParent(). + getMaxChildQueueResource(); if (max != null) { queue.setMaxShare(max); diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FairSchedulerConfiguration.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FairSchedulerConfiguration.java index 8e8e37b..37d0f12 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FairSchedulerConfiguration.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FairSchedulerConfiguration.java @@ -18,6 +18,7 @@ package org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair; import java.io.File; +import java.util.HashMap; import java.util.regex.Matcher; import java.util.regex.Pattern; @@ -29,6 +30,7 @@ import org.apache.hadoop.util.StringUtils; import org.apache.hadoop.yarn.api.records.Resource; import org.apache.hadoop.yarn.conf.YarnConfiguration; +import org.apache.hadoop.yarn.server.resourcemanager.resource.ResourceType; import org.apache.hadoop.yarn.server.utils.BuilderUtils; import org.apache.hadoop.yarn.util.resource.Resources; @@ -287,19 +289,61 @@ public float getReservableNodes() { * * @throws AllocationConfigurationException */ - public static Resource parseResourceConfigValue(String val) + public static ResourceConfiguration parseResourceConfigValue(String val) throws AllocationConfigurationException { + ResourceConfiguration resourceConfiguration; try { val = StringUtils.toLowerCase(val); - int memory = findResource(val, "mb"); - int vcores = findResource(val, "vcores"); - return BuilderUtils.newResource(memory, vcores); + if (val.contains("%")) { + resourceConfiguration = new ResourceConfiguration( + getResourcePercentage(val)); + } else { + int memory = findResource(val, "mb"); + int vcores = findResource(val, "vcores"); + resourceConfiguration = new ResourceConfiguration( + BuilderUtils.newResource(memory, vcores)); + } } catch (AllocationConfigurationException ex) { throw ex; } catch (Exception ex) { throw new AllocationConfigurationException( "Error reading resource config", ex); } + return resourceConfiguration; + } + + private static HashMap getResourcePercentage( + String val) throws AllocationConfigurationException { + HashMap resourcePercentage = new HashMap<>(); + String[] strings = val.split(","); + if (strings.length == 1) { + double percentage = findPercentage(strings[0], ""); + for (ResourceType resourceType: ResourceType.values()) { + resourcePercentage.put(resourceType, percentage/100); + } + } else { + double memPercentage = findPercentage(val, "memory"); + double vcorePercentage = findPercentage(val, "vcore"); + resourcePercentage.put(ResourceType.MEMORY, memPercentage/100); + resourcePercentage.put(ResourceType.CPU, vcorePercentage/100); + } + return resourcePercentage; + } + + private static double findPercentage(String val, String units) + throws AllocationConfigurationException { + Pattern pattern = Pattern.compile("((\\d+)(\\.\\d*)?)\\s*%\\s*" + units); + Matcher matcher = pattern.matcher(val); + if (!matcher.find()) { + if (units.equals("")) { + throw new AllocationConfigurationException("Invalid percentage: " + + val); + } else { + throw new AllocationConfigurationException("Missing resource: " + + units); + } + } + return Double.parseDouble(matcher.group(1)); } public long getUpdateInterval() { @@ -307,7 +351,7 @@ public long getUpdateInterval() { } private static int findResource(String val, String units) - throws AllocationConfigurationException { + throws AllocationConfigurationException { Pattern pattern = Pattern.compile("(\\d+)(\\.\\d*)?\\s*" + units); Matcher matcher = pattern.matcher(val); if (!matcher.find()) { diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/QueueManager.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/QueueManager.java index c08d13e..3de2e00 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/QueueManager.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/QueueManager.java @@ -41,7 +41,6 @@ import com.google.common.annotations.VisibleForTesting; import java.util.Iterator; import java.util.Set; -import org.apache.hadoop.yarn.api.records.Resource; /** * Maintains a list of queues as well as scheduling parameters for each queue, @@ -330,7 +329,7 @@ void setChildResourceLimits(FSParentQueue parent, FSQueue child, !configuredQueues.get(FSQueueType.PARENT).contains(child.getName())) { // For ad hoc queues, set their max reource allocations based on // their parents' default child settings. - Resource maxChild = parent.getMaxChildQueueResource(); + ResourceConfiguration maxChild = parent.getMaxChildQueueResource(); if (maxChild != null) { child.setMaxShare(maxChild); diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/ResourceConfiguration.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/ResourceConfiguration.java new file mode 100644 index 0000000..960019b --- /dev/null +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/ResourceConfiguration.java @@ -0,0 +1,71 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair; + +import org.apache.hadoop.classification.InterfaceAudience.Private; +import org.apache.hadoop.classification.InterfaceStability.Unstable; +import org.apache.hadoop.yarn.api.records.Resource; +import org.apache.hadoop.yarn.server.resourcemanager.resource.ResourceType; + +import java.util.HashMap; + +/** + * A ResourceConfiguration represents an entity that is used to configuration + * resources, such as maximum resources of a queue. It can be percentage of + * cluster resources or an absolute value. + */ +@Private +@Unstable +public class ResourceConfiguration { + private final boolean isPercentage; + private final Resource resource; + private final HashMap percentages; + + public boolean isPercentage() { + return isPercentage; + } + + public ResourceConfiguration(HashMap percentages) { + isPercentage = true; + this.percentages = percentages; + this.resource = null; + } + + public ResourceConfiguration(Resource resource) { + isPercentage = false; + this.percentages = null; + this.resource = resource; + } + + public Resource getResource(Resource clusterResource) { + if (isPercentage && clusterResource != null) { + long memory = (long) (clusterResource.getMemorySize() * + percentages.get(ResourceType.MEMORY)); + int vcore = (int) (clusterResource.getVirtualCores() * + percentages.get(ResourceType.CPU)); + return Resource.newInstance(memory, vcore); + } else { + return resource; + } + } + + public Resource getResource() { + return resource; + } +} diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestAllocationFileLoaderService.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestAllocationFileLoaderService.java index c8b9ad8..67b46f9 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestAllocationFileLoaderService.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestAllocationFileLoaderService.java @@ -235,21 +235,21 @@ public void testAllocationFileParsing() throws Exception { queueConf.getMinResources("root." + YarnConfiguration.DEFAULT_QUEUE_NAME)); assertEquals(Resources.createResource(2048, 10), - queueConf.getMaxResources("root.queueA")); + queueConf.getMaxResources("root.queueA").getResource()); assertEquals(Resources.createResource(5120, 110), - queueConf.getMaxResources("root.queueB")); - assertEquals(Resources.createResource(5120, 0), - queueConf.getMaxResources("root.queueC")); + queueConf.getMaxResources("root.queueB").getResource()); + assertEquals(Resources.createResource(4096, 100), + queueConf.getMaxResources("root.queueC").getResource()); assertEquals(Resources.createResource(4096, 100), - queueConf.getMaxResources("root.queueD")); + queueConf.getMaxResources("root.queueD").getResource()); assertEquals(Resources.createResource(4096, 100), - queueConf.getMaxResources("root.queueE")); + queueConf.getMaxResources("root.queueE").getResource()); assertEquals(Resources.createResource(4096, 100), - queueConf.getMaxResources("root.queueF")); + queueConf.getMaxResources("root.queueF").getResource()); assertEquals(Resources.createResource(4096, 100), - queueConf.getMaxResources("root.queueG")); + queueConf.getMaxResources("root.queueG").getResource()); assertEquals(Resources.createResource(4096, 100), - queueConf.getMaxResources("root.queueG.queueH")); + queueConf.getMaxResources("root.queueG.queueH").getResource()); assertEquals(Resources.createResource(1024, 0), queueConf.getMinResources("root.queueA")); @@ -279,9 +279,9 @@ public void testAllocationFileParsing() throws Exception { assertNull("Max child resources unexpectedly set for queue root.queueE", queueConf.getMaxChildResources("root.queueE")); assertEquals(Resources.createResource(2048, 64), - queueConf.getMaxChildResources("root.queueF")); + queueConf.getMaxChildResources("root.queueF").getResource()); assertEquals(Resources.createResource(2048, 64), - queueConf.getMaxChildResources("root.queueG")); + queueConf.getMaxChildResources("root.queueG").getResource()); assertNull("Max child resources unexpectedly set for " + "queue root.queueG.queueH", queueConf.getMaxChildResources("root.queueG.queueH")); diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFSLeafQueue.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFSLeafQueue.java index 2aed9bf..a5e37d1 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFSLeafQueue.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFSLeafQueue.java @@ -81,7 +81,7 @@ public void testUpdateDemand() { String queueName = "root.queue1"; FSLeafQueue schedulable = new FSLeafQueue(queueName, scheduler, null); - schedulable.setMaxShare(maxResource); + schedulable.setMaxShare(new ResourceConfiguration(maxResource)); assertEquals(schedulable.getMetrics().getMaxApps(), Integer.MAX_VALUE); assertEquals(schedulable.getMetrics().getSchedulingPolicy(), SchedulingPolicy.DEFAULT_POLICY.getName()); diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFSParentQueue.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFSParentQueue.java index 3ae8f83..671b94b 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFSParentQueue.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFSParentQueue.java @@ -19,6 +19,7 @@ package org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair; import org.apache.hadoop.yarn.util.SystemClock; +import org.apache.hadoop.yarn.util.resource.DefaultResourceCalculator; import org.junit.Before; import org.junit.Test; @@ -42,6 +43,8 @@ public void setUp() throws Exception { AllocationConfiguration allocConf = new AllocationConfiguration(conf); when(scheduler.getAllocationConfiguration()).thenReturn(allocConf); when(scheduler.getConf()).thenReturn(conf); + when(scheduler.getResourceCalculator()).thenReturn( + new DefaultResourceCalculator()); SystemClock clock = SystemClock.getInstance(); when(scheduler.getClock()).thenReturn(clock); notEmptyQueues = new HashSet(); diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFairScheduler.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFairScheduler.java index 0d54c33..52e18b1 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFairScheduler.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFairScheduler.java @@ -4608,7 +4608,8 @@ public void testMoveWouldViolateMaxResourcesConstraints() throws Exception { QueueManager queueMgr = scheduler.getQueueManager(); FSLeafQueue oldQueue = queueMgr.getLeafQueue("queue1", true); FSQueue queue2 = queueMgr.getLeafQueue("queue2", true); - queue2.setMaxShare(Resource.newInstance(1024, 1)); + queue2.setMaxShare( + new ResourceConfiguration(Resource.newInstance(1024, 1))); ApplicationAttemptId appAttId = createSchedulingRequest(1024, 1, "queue1", "user1", 3); @@ -5206,6 +5207,8 @@ public void testUpdateDemand() throws IOException { scheduler.reinitialize(conf, resourceManager.getRMContext()); Resource maxResource = Resources.createResource(1024 * 8); + ResourceConfiguration maxResourceConf = + new ResourceConfiguration(maxResource); FSAppAttempt app1 = mock(FSAppAttempt.class); Mockito.when(app1.getDemand()).thenReturn(maxResource); @@ -5217,15 +5220,15 @@ public void testUpdateDemand() throws IOException { FSLeafQueue aQueue = new FSLeafQueue("root.queue1.a", scheduler, queue1); - aQueue.setMaxShare(maxResource); + aQueue.setMaxShare(maxResourceConf); aQueue.addAppSchedulable(app1); FSLeafQueue bQueue = new FSLeafQueue("root.queue1.b", scheduler, queue1); - bQueue.setMaxShare(maxResource); + bQueue.setMaxShare(maxResourceConf); bQueue.addAppSchedulable(app2); - queue1.setMaxShare(maxResource); + queue1.setMaxShare(maxResourceConf); queue1.addChildQueue(aQueue); queue1.addChildQueue(bQueue); @@ -5263,7 +5266,7 @@ public void testDumpState() throws IOException { FSLeafQueue child1 = scheduler.getQueueManager().getLeafQueue("parent.child1", false); Resource resource = Resource.newInstance(4 * GB, 4); - child1.setMaxShare(resource); + child1.setMaxShare(new ResourceConfiguration(resource)); FSAppAttempt app = mock(FSAppAttempt.class); Mockito.when(app.getDemand()).thenReturn(resource); Mockito.when(app.getResourceUsage()).thenReturn(resource); @@ -5291,7 +5294,7 @@ public void testDumpState() throws IOException { assertTrue(child1.dumpState().equals(childQueueString)); FSParentQueue parent = scheduler.getQueueManager().getParentQueue("parent", false); - parent.setMaxShare(resource); + parent.setMaxShare(new ResourceConfiguration(resource)); parent.updateDemand(); String parentQueueString = "{Name: root.parent," diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFairSchedulerConfiguration.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFairSchedulerConfiguration.java index 8e7b666..da98a4b 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFairSchedulerConfiguration.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFairSchedulerConfiguration.java @@ -20,11 +20,7 @@ import static org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.FairSchedulerConfiguration.parseResourceConfigValue; import static org.junit.Assert.assertEquals; -import java.io.File; - -import org.junit.Assert; - -import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.yarn.api.records.Resource; import org.apache.hadoop.yarn.server.utils.BuilderUtils; import org.junit.Test; @@ -32,23 +28,59 @@ @Test public void testParseResourceConfigValue() throws Exception { assertEquals(BuilderUtils.newResource(1024, 2), - parseResourceConfigValue("2 vcores, 1024 mb")); + parseResourceConfigValue("2 vcores, 1024 mb").getResource()); + assertEquals(BuilderUtils.newResource(1024, 2), + parseResourceConfigValue("1024 mb, 2 vcores").getResource()); + assertEquals(BuilderUtils.newResource(1024, 2), + parseResourceConfigValue("2vcores,1024mb").getResource()); + assertEquals(BuilderUtils.newResource(1024, 2), + parseResourceConfigValue("1024mb,2vcores").getResource()); + assertEquals(BuilderUtils.newResource(1024, 2), + parseResourceConfigValue("1024 mb, 2 vcores").getResource()); + assertEquals(BuilderUtils.newResource(1024, 2), + parseResourceConfigValue("1024 Mb, 2 vCores").getResource()); + assertEquals(BuilderUtils.newResource(1024, 2), + parseResourceConfigValue(" 1024 mb, 2 vcores ").getResource()); + assertEquals(BuilderUtils.newResource(1024, 2), + parseResourceConfigValue(" 1024.3 mb, 2.35 vcores ").getResource()); assertEquals(BuilderUtils.newResource(1024, 2), - parseResourceConfigValue("1024 mb, 2 vcores")); + parseResourceConfigValue(" 1024. mb, 2. vcores ").getResource()); + + Resource clusterResource = BuilderUtils.newResource(2048, 4); assertEquals(BuilderUtils.newResource(1024, 2), - parseResourceConfigValue("2vcores,1024mb")); + parseResourceConfigValue("50% memory, 50% vcore"). + getResource(clusterResource)); assertEquals(BuilderUtils.newResource(1024, 2), - parseResourceConfigValue("1024mb,2vcores")); + parseResourceConfigValue("50% Memory, 50% VcOre"). + getResource(clusterResource)); assertEquals(BuilderUtils.newResource(1024, 2), - parseResourceConfigValue("1024 mb, 2 vcores")); + parseResourceConfigValue("50%").getResource(clusterResource)); + assertEquals(BuilderUtils.newResource(1024, 4), + parseResourceConfigValue("50% memory, 100% vcore"). + getResource(clusterResource)); + assertEquals(BuilderUtils.newResource(1024, 4), + parseResourceConfigValue(" 100% vcore, 50% memory"). + getResource(clusterResource)); + assertEquals(BuilderUtils.newResource(1024, 0), + parseResourceConfigValue("50% memory, 0% vcore"). + getResource(clusterResource)); assertEquals(BuilderUtils.newResource(1024, 2), - parseResourceConfigValue("1024 Mb, 2 vCores")); + parseResourceConfigValue("50 % memory, 50 % vcore"). + getResource(clusterResource)); assertEquals(BuilderUtils.newResource(1024, 2), - parseResourceConfigValue(" 1024 mb, 2 vcores ")); + parseResourceConfigValue("50%memory,50%vcore"). + getResource(clusterResource)); assertEquals(BuilderUtils.newResource(1024, 2), - parseResourceConfigValue(" 1024.3 mb, 2.35 vcores ")); + parseResourceConfigValue(" 50 % memory, 50 % vcore "). + getResource(clusterResource)); assertEquals(BuilderUtils.newResource(1024, 2), - parseResourceConfigValue(" 1024. mb, 2. vcores ")); + parseResourceConfigValue("50.% memory, 50.% vcore"). + getResource(clusterResource)); + + clusterResource = BuilderUtils.newResource(1024 * 10, 4); + assertEquals(BuilderUtils.newResource((int)(1024 * 10 * 0.109), 2), + parseResourceConfigValue("10.9% memory, 50.6% vcore"). + getResource(clusterResource)); } @Test(expected = AllocationConfigurationException.class) @@ -70,5 +102,14 @@ public void testOnlyCPU() throws Exception { public void testGibberish() throws Exception { parseResourceConfigValue("1o24vc0res"); } - + + @Test(expected = AllocationConfigurationException.class) + public void testNoUnitsPercentage() throws Exception { + parseResourceConfigValue("95%, 50% memory"); + } + + @Test(expected = AllocationConfigurationException.class) + public void testInvalidNumPercentage() throws Exception { + parseResourceConfigValue("95A% vcore, 50% memory"); + } } diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestMaxRunningAppsEnforcer.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestMaxRunningAppsEnforcer.java index f77df1e..964ecf5 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestMaxRunningAppsEnforcer.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestMaxRunningAppsEnforcer.java @@ -32,6 +32,7 @@ import org.apache.hadoop.yarn.api.records.ApplicationAttemptId; import org.apache.hadoop.yarn.api.records.ApplicationId; import org.apache.hadoop.yarn.util.ControlledClock; +import org.apache.hadoop.yarn.util.resource.DefaultResourceCalculator; import org.junit.Before; import org.junit.Test; @@ -55,7 +56,9 @@ public void setup() throws Exception { AllocationConfiguration allocConf = new AllocationConfiguration( conf); when(scheduler.getAllocationConfiguration()).thenReturn(allocConf); - + when(scheduler.getResourceCalculator()).thenReturn( + new DefaultResourceCalculator()); + queueManager = new QueueManager(scheduler); queueManager.initialize(conf); userMaxApps = allocConf.userMaxApps; diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestQueueManager.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestQueueManager.java index 05ab11c..6bf80d0 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestQueueManager.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestQueueManager.java @@ -24,6 +24,7 @@ import java.util.Set; import org.apache.hadoop.yarn.util.SystemClock; +import org.apache.hadoop.yarn.util.resource.DefaultResourceCalculator; import org.apache.hadoop.yarn.util.resource.Resources; import org.junit.Before; import org.junit.Test; @@ -50,6 +51,8 @@ public void setUp() throws Exception { when(scheduler.getAllocationConfiguration()).thenReturn(allocConf); when(scheduler.getConf()).thenReturn(conf); + when(scheduler.getResourceCalculator()).thenReturn( + new DefaultResourceCalculator()); SystemClock clock = SystemClock.getInstance(); @@ -206,7 +209,7 @@ public void testCreateQueueWithChildDefaults() { queueManager.updateAllocationConfiguration(allocConf); queueManager.getQueue("root.test").setMaxChildQueueResource( - Resources.createResource(8192, 256)); + new ResourceConfiguration(Resources.createResource(8192, 256))); FSQueue q1 = queueManager.createQueue("root.test.childC", FSQueueType.LEAF); assertNotNull("Leaf queue root.test.childC was not created", diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/TestFairSchedulerQueueInfo.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/TestFairSchedulerQueueInfo.java index 67d7340..83b7e41 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/TestFairSchedulerQueueInfo.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/TestFairSchedulerQueueInfo.java @@ -25,6 +25,7 @@ import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.FairSchedulerConfiguration; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.QueueManager; import org.apache.hadoop.yarn.util.SystemClock; +import org.apache.hadoop.yarn.util.resource.DefaultResourceCalculator; import org.junit.Assert; import org.junit.Test; @@ -43,6 +44,8 @@ public void testEmptyChildQueues() throws Exception { when(scheduler.getAllocationConfiguration()).thenReturn(allocConf); when(scheduler.getConf()).thenReturn(conf); when(scheduler.getClusterResource()).thenReturn(Resource.newInstance(1, 1)); + when(scheduler.getResourceCalculator()).thenReturn( + new DefaultResourceCalculator()); SystemClock clock = SystemClock.getInstance(); when(scheduler.getClock()).thenReturn(clock); QueueManager queueManager = new QueueManager(scheduler); diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/FairScheduler.md hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/FairScheduler.md index 748fda9..055ac50 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/FairScheduler.md +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/FairScheduler.md @@ -85,9 +85,9 @@ The allocation file must be in XML format. The format contains five types of ele * **minResources**: minimum resources the queue is entitled to, in the form "X mb, Y vcores". For the single-resource fairness policy, the vcores value is ignored. If a queue's minimum share is not satisfied, it will be offered available resources before any other queue under the same parent. Under the single-resource fairness policy, a queue is considered unsatisfied if its memory usage is below its minimum memory share. Under dominant resource fairness, a queue is considered unsatisfied if its usage for its dominant resource with respect to the cluster capacity is below its minimum share for that resource. If multiple queues are unsatisfied in this situation, resources go to the queue with the smallest ratio between relevant resource usage and minimum. Note that it is possible that a queue that is below its minimum may not immediately get up to its minimum when it submits an application, because already-running jobs may be using those resources. - * **maxResources**: maximum resources a queue is allowed, in the form "X mb, Y vcores". A queue will never be assigned a container that would put its aggregate usage over this limit. + * **maxResources**: maximum resources a queue is allowed, in the form "X mb, Y vcores". A queue will never be assigned a container that would put its aggregate usage over this limit. Set it to a percentage of all cluster resources in form of "50%" or "50% memory, 60% vcore". - * **maxChildResources**: maximum resources an ad hoc child queue is allowed, in the form "X mb, Y vcores". Any ad hoc queue that is a direct child of a queue with this property set will have it's maxResources property set accordingly. + * **maxChildResources**: maximum resources an ad hoc child queue is allowed, in the form "X mb, Y vcores". Any ad hoc queue that is a direct child of a queue with this property set will have it's maxResources property set accordingly. Set it to a percentage of all cluster resources in form of "50%" or "50% memory, 60% vcore". * **maxRunningApps**: limit the number of apps from the queue to run at once