diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/AllocationConfiguration.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/AllocationConfiguration.java index 54dd090617d..b671c806576 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/AllocationConfiguration.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/AllocationConfiguration.java @@ -91,6 +91,8 @@ private final SchedulingPolicy defaultSchedulingPolicy; + private final Map queueMaxContainerResourcesMap; + // Policy for mapping apps to queues @VisibleForTesting QueuePlacementPolicy placementPolicy; @@ -138,6 +140,7 @@ public AllocationConfiguration(QueueProperties queueProperties, this.placementPolicy = newPlacementPolicy; this.configuredQueues = queueProperties.getConfiguredQueues(); this.nonPreemptableQueues = queueProperties.getNonPreemptableQueues(); + this.queueMaxContainerResourcesMap = queueProperties.getMaxContainerResources(); } public AllocationConfiguration(Configuration conf) { @@ -167,6 +170,7 @@ public AllocationConfiguration(Configuration conf) { placementPolicy = QueuePlacementPolicy.fromConfiguration(conf, configuredQueues); nonPreemptableQueues = new HashSet<>(); + queueMaxContainerResourcesMap = new HashMap<>(); } /** @@ -272,6 +276,12 @@ ConfigurableResource getMaxResources(String queue) { return maxQueueResource; } + @VisibleForTesting + Resource getQueueMaxContainerResources(String queue) { + Resource resource = queueMaxContainerResourcesMap.get(queue); + return resource == null ? Resources.none() : resource; + } + /** * Get the maximum resource allocation for children of the given queue. * @@ -375,11 +385,13 @@ public void initFSQueue(FSQueue queue){ queue.setMaxRunningApps(getQueueMaxApps(name)); queue.setMaxAMShare(getQueueMaxAMShare(name)); queue.setMaxChildQueueResource(getMaxChildResources(name)); + queue.setMaxContainerResources(getQueueMaxContainerResources(name)); // Set queue metrics. queue.getMetrics().setMinShare(queue.getMinShare()); queue.getMetrics().setMaxShare(queue.getMaxShare()); queue.getMetrics().setMaxApps(queue.getMaxRunningApps()); queue.getMetrics().setSchedulingPolicy(getSchedulingPolicy(name).getName()); + queue.getMetrics().setMaxContainerResources(queue.getMaximumResourceCapability()); } } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSLeafQueue.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSLeafQueue.java index e7da16f59ac..0789a29efec 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSLeafQueue.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSLeafQueue.java @@ -547,6 +547,15 @@ public void setWeights(float weight) { this.weights = weight; } + @Override + public Resource getMaximumResourceCapability() { + if(maxContainerResources.equals(Resources.none()) && getParent() != null) { + return getParent().getMaximumResourceCapability(); + } else { + return maxContainerResources; + } + } + /** * Helper method to compute the amount of minshare starvation. * diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSParentQueue.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSParentQueue.java index 26c5630a6d5..f831d0f04ba 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSParentQueue.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSParentQueue.java @@ -59,7 +59,19 @@ public FSParentQueue(String name, FairScheduler scheduler, FSParentQueue parent) { super(name, scheduler, parent); } - + + @Override + public Resource getMaximumResourceCapability() { + if (getName().equals("root")) { + return maxContainerResources; + } + if(maxContainerResources.equals(Resources.none()) && getParent() != null) { + return getParent().getMaximumResourceCapability(); + } else { + return maxContainerResources; + } + } + void addChildQueue(FSQueue child) { writeLock.lock(); try { diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSQueue.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSQueue.java index 6b88a329fa3..04ab716e5dc 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSQueue.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSQueue.java @@ -84,6 +84,7 @@ private float fairSharePreemptionThreshold = 0.5f; private boolean preemptable = true; private boolean isDynamic = true; + protected Resource maxContainerResources; public FSQueue(String name, FairScheduler scheduler, FSParentQueue parent) { this.name = name; @@ -163,6 +164,12 @@ public void setMaxShare(ConfigurableResource maxShare){ this.maxShare = maxShare; } + public void setMaxContainerResources(Resource maxContainerResources){ + this.maxContainerResources = maxContainerResources; + } + + public abstract Resource getMaximumResourceCapability(); + @Override public Resource getMaxShare() { Resource maxResource = maxShare.getResource(scheduler.getClusterResource()); @@ -579,7 +586,6 @@ public String dumpState() { return sb.toString(); } - /** * Recursively dump states of all queues. * @@ -594,4 +600,5 @@ public boolean isDynamic() { public void setDynamic(boolean dynamic) { this.isDynamic = dynamic; } + } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSQueueMetrics.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSQueueMetrics.java index 4fe3973f7f7..3af087d91c6 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSQueueMetrics.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSQueueMetrics.java @@ -33,19 +33,36 @@ @Metrics(context="yarn") public class FSQueueMetrics extends QueueMetrics { - @Metric("Fair share of memory in MB") MutableGaugeLong fairShareMB; - @Metric("Fair share of CPU in vcores") MutableGaugeLong fairShareVCores; - @Metric("Steady fair share of memory in MB") MutableGaugeLong steadyFairShareMB; - @Metric("Steady fair share of CPU in vcores") MutableGaugeLong steadyFairShareVCores; - @Metric("Minimum share of memory in MB") MutableGaugeLong minShareMB; - @Metric("Minimum share of CPU in vcores") MutableGaugeLong minShareVCores; - @Metric("Maximum share of memory in MB") MutableGaugeLong maxShareMB; - @Metric("Maximum share of CPU in vcores") MutableGaugeLong maxShareVCores; - @Metric("Maximum number of applications") MutableGaugeInt maxApps; - @Metric("Maximum AM share of memory in MB") MutableGaugeLong maxAMShareMB; - @Metric("Maximum AM share of CPU in vcores") MutableGaugeInt maxAMShareVCores; - @Metric("AM resource usage of memory in MB") MutableGaugeLong amResourceUsageMB; - @Metric("AM resource usage of CPU in vcores") MutableGaugeInt amResourceUsageVCores; + @Metric("Fair share of memory in MB") + MutableGaugeLong fairShareMB; + @Metric("Fair share of CPU in vcores") + MutableGaugeLong fairShareVCores; + @Metric("Steady fair share of memory in MB") + MutableGaugeLong steadyFairShareMB; + @Metric("Steady fair share of CPU in vcores") + MutableGaugeLong steadyFairShareVCores; + @Metric("Minimum share of memory in MB") + MutableGaugeLong minShareMB; + @Metric("Minimum share of CPU in vcores") + MutableGaugeLong minShareVCores; + @Metric("Maximum share of memory in MB") + MutableGaugeLong maxShareMB; + @Metric("Maximum share of CPU in vcores") + MutableGaugeLong maxShareVCores; + @Metric("Maximum number of applications") + MutableGaugeInt maxApps; + @Metric("Maximum AM share of memory in MB") + MutableGaugeLong maxAMShareMB; + @Metric("Maximum AM share of CPU in vcores") + MutableGaugeInt maxAMShareVCores; + @Metric("Maximum resource usage of memory in MB for a container") + MutableGaugeLong maxContainerResourceUsageMB; + @Metric("Maximum resource usage of CPU in vcores for a container") + MutableGaugeLong maxContainerResourceUsageVCores; + @Metric("AM resource usage of memory in MB") + MutableGaugeLong amResourceUsageMB; + @Metric("AM resource usage of CPU in vcores") + MutableGaugeInt amResourceUsageVCores; private String schedulingPolicy; @@ -170,6 +187,11 @@ public void setAMResourceUsage(Resource resource) { amResourceUsageVCores.set(resource.getVirtualCores()); } + public void setMaxContainerResources(Resource resources) { + maxContainerResourceUsageMB.set(resources.getMemorySize()); + maxContainerResourceUsageVCores.set(resources.getVirtualCores()); + } + /** * Get the scheduling policy. * diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FairScheduler.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FairScheduler.java index eb9f6af7101..e91c357cf06 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FairScheduler.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FairScheduler.java @@ -90,6 +90,7 @@ import org.apache.hadoop.yarn.server.resourcemanager.scheduler.event.ReleaseContainerEvent; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.event.SchedulerEvent; +import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.allocation.QueueMaxContainerAllocationValidator; import org.apache.hadoop.yarn.server.resourcemanager.security.RMContainerTokenSecretManager; import org.apache.hadoop.yarn.util.resource.DefaultResourceCalculator; import org.apache.hadoop.yarn.util.resource.DominantResourceCalculator; @@ -229,7 +230,7 @@ private void validateConf(FairSchedulerConfiguration config) { YarnConfiguration.RM_SCHEDULER_MINIMUM_ALLOCATION_MB, YarnConfiguration.DEFAULT_RM_SCHEDULER_MINIMUM_ALLOCATION_MB); int maxMem = config.getInt( - YarnConfiguration.RM_SCHEDULER_MAXIMUM_ALLOCATION_MB, + YarnConfiguration.RM_SCHEDULER_MAXIMUM_ALLOCATION_MB, YarnConfiguration.DEFAULT_RM_SCHEDULER_MAXIMUM_ALLOCATION_MB); if (minMem < 0 || minMem > maxMem) { @@ -256,7 +257,7 @@ private void validateConf(FairSchedulerConfiguration config) { YarnConfiguration.RM_SCHEDULER_MINIMUM_ALLOCATION_VCORES, YarnConfiguration.DEFAULT_RM_SCHEDULER_MINIMUM_ALLOCATION_VCORES); int maxVcores = config.getInt( - YarnConfiguration.RM_SCHEDULER_MAXIMUM_ALLOCATION_VCORES, + YarnConfiguration.RM_SCHEDULER_MAXIMUM_ALLOCATION_VCORES, YarnConfiguration.DEFAULT_RM_SCHEDULER_MAXIMUM_ALLOCATION_VCORES); if (minVcores < 0 || minVcores > maxVcores) { @@ -839,6 +840,20 @@ public Resource getNormalizedResource(Resource requestedResource) { incrAllocation); } + @Override + public Resource getMaximumResourceCapability(String queueName) { + FSQueue queue = queueMgr.getQueue(queueName); + if (queue == null) { + return getMaximumResourceCapability(); + } + Resource queueMaxResourceCapability = queue.getMaximumResourceCapability(); + if (queueMaxResourceCapability.equals(Resources.none())) { + return getMaximumResourceCapability(); + } else { + return queueMaxResourceCapability; + } + } + @VisibleForTesting @Override public void killContainer(RMContainer container) { @@ -1446,6 +1461,7 @@ private void initScheduler(Configuration conf) throws IOException { } catch (Exception e) { throw new IOException("Failed to initialize FairScheduler", e); } + new QueueMaxContainerAllocationValidator(queueMgr, conf).validate(); } @VisibleForTesting diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/allocation/AllocationFileQueueParser.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/allocation/AllocationFileQueueParser.java index 441c34a1aa1..170dc566051 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/allocation/AllocationFileQueueParser.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/allocation/AllocationFileQueueParser.java @@ -51,6 +51,7 @@ private static final String MAX_CHILD_RESOURCES = "maxChildResources"; private static final String MAX_RUNNING_APPS = "maxRunningApps"; private static final String MAX_AMSHARE = "maxAMShare"; + public static final String MAX_CONTAINER_RESOURCES = "maxContainerResources"; private static final String WEIGHT = "weight"; private static final String MIN_SHARE_PREEMPTION_TIMEOUT = "minSharePreemptionTimeout"; @@ -155,6 +156,11 @@ private void loadQueue(String parentName, Element element, float val = Float.parseFloat(text); val = Math.min(val, 1.0f); builder.queueMaxAMShares(queueName, val); + } else if (MAX_CONTAINER_RESOURCES.equals(field.getTagName())) { + String text = getTrimmedTextData(field); + ConfigurableResource val = + FairSchedulerConfiguration.parseResourceConfigValue(text); + builder.queueMaxContainerResources(queueName, val.getResource()); } else if (WEIGHT.equals(field.getTagName())) { String text = getTrimmedTextData(field); double val = Double.parseDouble(text); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/allocation/QueueMaxContainerAllocationValidator.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/allocation/QueueMaxContainerAllocationValidator.java new file mode 100644 index 00000000000..a9ab3c3fd66 --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/allocation/QueueMaxContainerAllocationValidator.java @@ -0,0 +1,99 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * http://www.apache.org/licenses/LICENSE-2.0 + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.allocation; + +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.yarn.api.records.Resource; +import org.apache.hadoop.yarn.conf.YarnConfiguration; +import org.apache.hadoop.yarn.exceptions.YarnRuntimeException; +import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.FSQueue; +import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.FairScheduler; +import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.QueueManager; +import org.apache.hadoop.yarn.util.resource.Resources; + +import static org.apache.hadoop.yarn.conf.YarnConfiguration.RM_SCHEDULER_MAXIMUM_ALLOCATION_MB; +import static org.apache.hadoop.yarn.conf.YarnConfiguration.RM_SCHEDULER_MAXIMUM_ALLOCATION_VCORES; +import static org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.allocation.AllocationFileQueueParser.MAX_CONTAINER_RESOURCES; + +public class QueueMaxContainerAllocationValidator { + + private static final Log LOG = LogFactory.getLog(QueueMaxContainerAllocationValidator.class); + + private QueueManager queueManager; + private Configuration config; + + public QueueMaxContainerAllocationValidator(QueueManager queueManager, + Configuration config) { + this.queueManager = queueManager; + this.config = config; + } + + public static String createExceptionText(String propertyName, FSQueue queue, + int maxResourceValue) { + return "Invalid queue resource allocation! " + "Property " + + MAX_CONTAINER_RESOURCES + "=" + queue.getMaximumResourceCapability() + + " at queue: " + queue.getName() + " cannot be higher than " + + propertyName + "=" + maxResourceValue; + } + + public void validate() { + Resource resource = + queueManager.getQueue("root").getMaximumResourceCapability(); + if (!resource.equals(Resources.none())) { + throw new YarnRuntimeException( + "Invalid queue resource allocation," + " property" + + MAX_CONTAINER_RESOURCES + " is invalid for root queue!"); + } + int maxVcores = config.getInt(RM_SCHEDULER_MAXIMUM_ALLOCATION_VCORES, + YarnConfiguration.DEFAULT_RM_SCHEDULER_MAXIMUM_ALLOCATION_VCORES); + int maxMem = config.getInt(RM_SCHEDULER_MAXIMUM_ALLOCATION_MB, + YarnConfiguration.DEFAULT_RM_SCHEDULER_MAXIMUM_ALLOCATION_MB); + + LOG.info("MaxMem:" + maxMem); + LOG.info("MaxCores:" + maxVcores); + + queueManager.getQueues().forEach(fsQueue -> { + checkContainerResources(maxVcores, maxMem, fsQueue); + }); + } + + private void checkContainerResources(int maxVcores, int maxMem, + FSQueue fsQueue) { + long memorySize = fsQueue.getMaximumResourceCapability().getMemorySize(); + int virtualCores = fsQueue.getMaximumResourceCapability().getVirtualCores(); + + LOG.info("queueMem:" + memorySize); + LOG.info("queueCores:" + virtualCores); + + if (maxMem < memorySize) { + throwException(RM_SCHEDULER_MAXIMUM_ALLOCATION_MB, fsQueue, maxMem); + } + + if (maxVcores < virtualCores) { + throwException(RM_SCHEDULER_MAXIMUM_ALLOCATION_VCORES, fsQueue, + maxVcores); + } + } + + private void throwException(String propertyName, FSQueue queue, + int maxResourceValue) { + throw new YarnRuntimeException( + createExceptionText(propertyName, queue, maxResourceValue)); + } +} diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/allocation/QueueProperties.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/allocation/QueueProperties.java index ee5f1790237..2087e061ceb 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/allocation/QueueProperties.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/allocation/QueueProperties.java @@ -53,6 +53,7 @@ private final Set reservableQueues; private final Set nonPreemptableQueues; private final Map> configuredQueues; + private final Map queueMaxContainerResources; QueueProperties(Builder builder) { this.reservableQueues = builder.reservableQueues; @@ -70,6 +71,7 @@ this.maxChildQueueResources = builder.maxChildQueueResources; this.reservationAcls = builder.reservationAcls; this.queueAcls = builder.queueAcls; + this.queueMaxContainerResources = builder.queueMaxContainerResources; } public Map> getConfiguredQueues() { @@ -133,7 +135,11 @@ return nonPreemptableQueues; } - /** + public Map getMaxContainerResources() { + return queueMaxContainerResources; + } + + /** * Builder class for {@link QueueProperties}. * All methods are adding queue properties to the maps of this builder * keyed by the queue's name except some methods @@ -149,6 +155,7 @@ new HashMap<>(); private Map queueMaxApps = new HashMap<>(); private Map queueMaxAMShares = new HashMap<>(); + private Map queueMaxContainerResources = new HashMap<>(); private Map queueWeights = new HashMap<>(); private Map queuePolicies = new HashMap<>(); private Map minSharePreemptionTimeouts = new HashMap<>(); @@ -253,6 +260,11 @@ public Builder nonPreemptableQueues(String queue) { return this; } + public Builder queueMaxContainerResources(String queueName, Resource value) { + queueMaxContainerResources.put(queueName, value); + return this; + } + public void configuredQueues(FSQueueType queueType, String queueName) { this.configuredQueues.get(queueType).add(queueName); } @@ -275,6 +287,5 @@ public boolean isAclDefinedForAccessType(String queueName, public QueueProperties build() { return new QueueProperties(this); } - } } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/FairSchedulerPage.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/FairSchedulerPage.java index ef417d4760f..e343a45d41b 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/FairSchedulerPage.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/FairSchedulerPage.java @@ -78,6 +78,8 @@ protected void render(Block html) { __("Num Pending Applications:", qinfo.getNumPendingApplications()). __("Min Resources:", qinfo.getMinResources().toString()). __("Max Resources:", qinfo.getMaxResources().toString()). + __("Max Container Resources:", + qinfo.getMaxContainerResources().toString()). __("Reserved Resources:", qinfo.getReservedResources().toString()); int maxApps = qinfo.getMaxApplications(); if (maxApps < Integer.MAX_VALUE) { @@ -107,6 +109,8 @@ protected void render(Block html) { __("Used Resources:", qinfo.getUsedResources().toString()). __("Min Resources:", qinfo.getMinResources().toString()). __("Max Resources:", qinfo.getMaxResources().toString()). + __("Max Container Resources:", + qinfo.getMaxContainerResources().toString()). __("Reserved Resources:", qinfo.getReservedResources().toString()); int maxApps = qinfo.getMaxApplications(); if (maxApps < Integer.MAX_VALUE) { diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/FairSchedulerQueueInfo.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/FairSchedulerQueueInfo.java index 913513c52ae..4862b82c339 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/FairSchedulerQueueInfo.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/FairSchedulerQueueInfo.java @@ -60,6 +60,7 @@ private ResourceInfo fairResources; private ResourceInfo clusterResources; private ResourceInfo reservedResources; + private ResourceInfo maxContainerResources; private long allocatedContainers; private long reservedContainers; @@ -99,6 +100,7 @@ public FairSchedulerQueueInfo(FSQueue queue, FairScheduler scheduler) { maxResources = new ResourceInfo( Resources.componentwiseMin(queue.getMaxShare(), scheduler.getClusterResource())); + maxContainerResources = new ResourceInfo(scheduler.getMaximumResourceCapability(queueName)); reservedResources = new ResourceInfo(queue.getReservedResource()); fractionMemSteadyFairShare = @@ -186,7 +188,11 @@ public ResourceInfo getMinResources() { public ResourceInfo getMaxResources() { return maxResources; } - + + public ResourceInfo getMaxContainerResources() { + return maxContainerResources; + } + public ResourceInfo getReservedResources() { return reservedResources; } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FairSchedulerTestBase.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FairSchedulerTestBase.java index 3ac3849cf73..81a5e4d9138 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FairSchedulerTestBase.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FairSchedulerTestBase.java @@ -77,6 +77,7 @@ public static final float TEST_RESERVATION_THRESHOLD = 0.09f; private static final int SLEEP_DURATION = 10; private static final int SLEEP_RETRIES = 1000; + protected static final int RM_SCHEDULER_MAXIMUM_ALLOCATION_MB_VALUE = 10240; final static ContainerUpdates NULL_UPDATE_REQUESTS = new ContainerUpdates(); @@ -93,7 +94,7 @@ public Configuration createConfiguration() { conf.setInt(YarnConfiguration.RM_SCHEDULER_MINIMUM_ALLOCATION_MB, 0); conf.setInt(FairSchedulerConfiguration.RM_SCHEDULER_INCREMENT_ALLOCATION_MB, 1024); - conf.setInt(YarnConfiguration.RM_SCHEDULER_MAXIMUM_ALLOCATION_MB, 10240); + conf.setInt(YarnConfiguration.RM_SCHEDULER_MAXIMUM_ALLOCATION_MB, RM_SCHEDULER_MAXIMUM_ALLOCATION_MB_VALUE); conf.setBoolean(FairSchedulerConfiguration.ASSIGN_MULTIPLE, false); conf.setLong(FairSchedulerConfiguration.UPDATE_INTERVAL_MS, 10); conf.setFloat(FairSchedulerConfiguration.PREEMPTION_THRESHOLD, 0f); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestAllocationFileLoaderService.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestAllocationFileLoaderService.java index 50a003ecd11..3078d289da0 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestAllocationFileLoaderService.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestAllocationFileLoaderService.java @@ -23,21 +23,21 @@ import org.apache.hadoop.fs.Path; import org.apache.hadoop.fs.UnsupportedFileSystemException; import org.apache.hadoop.hdfs.MiniDFSCluster; +import org.apache.hadoop.yarn.LocalConfigurationProvider; +import org.apache.hadoop.yarn.api.records.Resource; import org.apache.hadoop.yarn.conf.YarnConfiguration; +import org.apache.hadoop.yarn.exceptions.YarnException; import org.apache.hadoop.yarn.server.resourcemanager.reservation.ReservationSchedulerConfiguration; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.QueuePlacementRule.NestedUserQueue; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.allocationfile.AllocationFileWriter; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.policies.DominantResourceFairnessPolicy; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.policies.FairSharePolicy; import org.apache.hadoop.yarn.util.ControlledClock; +import org.apache.hadoop.yarn.util.resource.ResourceUtils; import org.apache.hadoop.yarn.util.resource.Resources; import org.junit.Test; -import java.io.File; -import java.io.FileOutputStream; -import java.io.FileWriter; -import java.io.IOException; -import java.io.OutputStreamWriter; -import java.io.PrintWriter; + +import java.io.*; import java.net.URISyntaxException; import java.net.URL; import java.nio.charset.StandardCharsets; @@ -53,6 +53,33 @@ public class TestAllocationFileLoaderService { + private static final String A_CUSTOM_RESOURCE = "a-custom-resource"; + + private static class CustomResourceTypesConfigurationProvider + extends LocalConfigurationProvider { + + @Override + public InputStream getConfigurationInputStream(Configuration bootstrapConf, + String name) throws YarnException, IOException { + if (YarnConfiguration.RESOURCE_TYPES_CONFIGURATION_FILE.equals(name)) { + return new ByteArrayInputStream(( + "\n" + + " \n" + + " yarn.resource-types\n" + + " " + A_CUSTOM_RESOURCE + "\n" + + " \n" + + " \n" + + " yarn.resource-types.a-custom-resource.units\n" + + " k\n" + + " \n" + + "\n") + .getBytes()); + } else { + return super.getConfigurationInputStream(bootstrapConf, name); + } + } + } + final static String TEST_DIR = new File(System.getProperty("test.build.data", "/tmp")).getAbsolutePath(); @@ -204,6 +231,11 @@ public void testReload() throws Exception { @Test public void testAllocationFileParsing() throws Exception { Configuration conf = new Configuration(); + + conf.set(YarnConfiguration.RM_CONFIGURATION_PROVIDER_CLASS, + CustomResourceTypesConfigurationProvider.class.getName()); + ResourceUtils.resetResourceTypes(conf); + conf.set(FairSchedulerConfiguration.ALLOCATION_FILE, ALLOC_FILE); AllocationFileLoaderService allocLoader = new AllocationFileLoaderService(); @@ -246,6 +278,8 @@ public void testAllocationFileParsing() throws Exception { .fairSharePreemptionTimeout(120) .minSharePreemptionTimeout(50) .fairSharePreemptionThreshold(0.6) + .maxContainerResources( + "vcores=16, memory-mb=512, " + A_CUSTOM_RESOURCE + "=10") // Create hierarchical queues G,H, with different min/fair // share preemption timeouts and preemption thresholds. // Also add a child default to make sure it doesn't impact queue H. @@ -253,6 +287,7 @@ public void testAllocationFileParsing() throws Exception { .fairSharePreemptionTimeout(180) .minSharePreemptionTimeout(40) .fairSharePreemptionThreshold(0.7) + .maxContainerResources("1024mb,8vcores") .buildSubQueue() .buildQueue() // Set default limit of apps per queue to 15 @@ -286,8 +321,6 @@ public void testAllocationFileParsing() throws Exception { assertEquals(6, queueConf.getConfiguredQueues().get(FSQueueType.LEAF).size()); assertEquals(Resources.createResource(0), queueConf.getMinResources("root." + YarnConfiguration.DEFAULT_QUEUE_NAME)); - assertEquals(Resources.createResource(0), - queueConf.getMinResources("root." + YarnConfiguration.DEFAULT_QUEUE_NAME)); assertEquals(Resources.createResource(2048, 10), queueConf.getMaxResources("root.queueA").getResource()); @@ -358,6 +391,28 @@ public void testAllocationFileParsing() throws Exception { assertEquals(.4f, queueConf.getQueueMaxAMShare("root.queueD"), 0.01); assertEquals(.5f, queueConf.getQueueMaxAMShare("root.queueE"), 0.01); + Resource expectedResourceWithCustomType = Resources.createResource(512, 16); + expectedResourceWithCustomType.setResourceValue(A_CUSTOM_RESOURCE, 10); + + assertEquals(Resources.none(), queueConf.getQueueMaxContainerResources( + "root." + YarnConfiguration.DEFAULT_QUEUE_NAME)); + assertEquals(Resources.none(), + queueConf.getQueueMaxContainerResources("root.queueA")); + assertEquals(Resources.none(), + queueConf.getQueueMaxContainerResources("root.queueB")); + assertEquals(Resources.none(), + queueConf.getQueueMaxContainerResources("root.queueC")); + assertEquals(Resources.none(), + queueConf.getQueueMaxContainerResources("root.queueD")); + assertEquals(Resources.none(), + queueConf.getQueueMaxContainerResources("root.queueE")); + assertEquals(Resources.none(), + queueConf.getQueueMaxContainerResources("root.queueF")); + assertEquals(expectedResourceWithCustomType, + queueConf.getQueueMaxContainerResources("root.queueG")); + assertEquals(Resources.createResource(1024, 8), + queueConf.getQueueMaxContainerResources("root.queueG.queueH")); + assertEquals(120000, queueConf.getMinSharePreemptionTimeout("root")); assertEquals(-1, queueConf.getMinSharePreemptionTimeout("root." + YarnConfiguration.DEFAULT_QUEUE_NAME)); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFairScheduler.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFairScheduler.java index 9120d3a6cc1..1675dfc5714 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFairScheduler.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFairScheduler.java @@ -18,6 +18,7 @@ package org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair; +import static org.apache.hadoop.yarn.conf.YarnConfiguration.DEFAULT_RM_SCHEDULER_MAXIMUM_ALLOCATION_VCORES; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertFalse; import static org.junit.Assert.assertNotEquals; @@ -198,7 +199,53 @@ public void testConfValidation() throws Exception { } } - // TESTS + @Test(expected = YarnRuntimeException.class) + public void validateRootQueueMaxContainerAllocationTest() throws IOException{ + PrintWriter out = new PrintWriter(new FileWriter(ALLOC_FILE)); + out.println(""); + out.println(""); + out.println(" "); + out.println(" 1024 mb 1 vcores"); + out.println(" "); + out.println(""); + out.close(); + + conf.set(FairSchedulerConfiguration.ALLOCATION_FILE, ALLOC_FILE); + scheduler.init(conf); + } + + @Test(expected = YarnRuntimeException.class) + public void tooHighMemoryMaxContainerAllocationTest() throws IOException{ + int tooHighAllocation = RM_SCHEDULER_MAXIMUM_ALLOCATION_MB_VALUE + 1; + PrintWriter out = new PrintWriter(new FileWriter(ALLOC_FILE)); + out.println(""); + out.println(""); + out.println(" "); + out.println(" " + tooHighAllocation + " mb 1 vcores"); + out.println(" "); + out.println(""); + out.close(); + + conf.set(FairSchedulerConfiguration.ALLOCATION_FILE, ALLOC_FILE); + scheduler.init(conf); + } + + @Test(expected = YarnRuntimeException.class) + public void tooManyVCoresMaxContainerAllocationTest() throws IOException{ + int toManyVcores = conf.getInt(YarnConfiguration.RM_SCHEDULER_MAXIMUM_ALLOCATION_VCORES,1 ) + 1; + PrintWriter out = new PrintWriter(new FileWriter(ALLOC_FILE)); + out.println(""); + out.println(""); + out.println(" "); + out.println(" 1024 mb " + toManyVcores + + " vcores"); + out.println(" "); + out.println(""); + out.close(); + + conf.set(FairSchedulerConfiguration.ALLOCATION_FILE, ALLOC_FILE); + scheduler.init(conf); + } @SuppressWarnings("deprecation") @Test(timeout=2000) @@ -338,6 +385,53 @@ public void testSimpleFairShareCalculation() throws IOException { } } + @Test + public void testQueueMaximumCapacityAllocations() throws IOException { + conf.set(FairSchedulerConfiguration.ALLOCATION_FILE, ALLOC_FILE); + + PrintWriter out = new PrintWriter(new FileWriter(ALLOC_FILE)); + out.println(""); + out.println(""); + out.println(" "); + out.println( + " 512 mb 1 vcores"); + out.println(" "); + out.println(" "); + out.println(" "); + out.println(" "); + out.println( + " 2048 mb 3 vcores"); + out.println(" "); + out.println(" "); + out.println(" "); + out.println(""); + out.close(); + out.close(); + + scheduler.init(conf); + + Assert.assertEquals(1, scheduler.getMaximumResourceCapability("root.queueA") + .getVirtualCores()); + Assert.assertEquals(512, + scheduler.getMaximumResourceCapability("root.queueA").getMemorySize()); + + Assert.assertEquals(DEFAULT_RM_SCHEDULER_MAXIMUM_ALLOCATION_VCORES, + scheduler.getMaximumResourceCapability("root.queueB") + .getVirtualCores()); + Assert.assertEquals(RM_SCHEDULER_MAXIMUM_ALLOCATION_MB_VALUE, + scheduler.getMaximumResourceCapability("root.queueB").getMemorySize()); + + Assert.assertEquals(3, scheduler.getMaximumResourceCapability("root.queueC") + .getVirtualCores()); + Assert.assertEquals(2048, + scheduler.getMaximumResourceCapability("root.queueC").getMemorySize()); + + Assert.assertEquals(3, scheduler + .getMaximumResourceCapability("root.queueC.queueD").getVirtualCores()); + Assert.assertEquals(2048, scheduler + .getMaximumResourceCapability("root.queueC.queueD").getMemorySize()); + } + /** * Test fair shares when max resources are set but are too high to impact * the shares. @@ -1316,7 +1410,7 @@ public void testRackLocalAppReservationThreshold() throws Exception { // New node satisfies resource request scheduler.update(); scheduler.handle(new NodeUpdateSchedulerEvent(node4)); - assertEquals(10240, scheduler.getQueueManager().getQueue("queue1"). + assertEquals(RM_SCHEDULER_MAXIMUM_ALLOCATION_MB_VALUE, scheduler.getQueueManager().getQueue("queue1"). getResourceUsage().getMemorySize()); scheduler.handle(new NodeUpdateSchedulerEvent(node1)); @@ -4100,10 +4194,10 @@ public void testQueueMaxAMShareWithContainerReservation() throws Exception { scheduler.reinitialize(conf, resourceManager.getRMContext()); RMNode node1 = - MockNodes.newNodeInfo(1, Resources.createResource(10240, 10), + MockNodes.newNodeInfo(1, Resources.createResource(RM_SCHEDULER_MAXIMUM_ALLOCATION_MB_VALUE, 10), 1, "127.0.0.1"); RMNode node2 = - MockNodes.newNodeInfo(1, Resources.createResource(10240, 10), + MockNodes.newNodeInfo(1, Resources.createResource(RM_SCHEDULER_MAXIMUM_ALLOCATION_MB_VALUE, 10), 2, "127.0.0.2"); RMNode node3 = MockNodes.newNodeInfo(1, Resources.createResource(5120, 5), @@ -4122,10 +4216,10 @@ public void testQueueMaxAMShareWithContainerReservation() throws Exception { true); Resource amResource1 = Resource.newInstance(1024, 1); Resource amResource2 = Resource.newInstance(1024, 1); - Resource amResource3 = Resource.newInstance(10240, 1); + Resource amResource3 = Resource.newInstance(RM_SCHEDULER_MAXIMUM_ALLOCATION_MB_VALUE, 1); Resource amResource4 = Resource.newInstance(5120, 1); Resource amResource5 = Resource.newInstance(1024, 1); - Resource amResource6 = Resource.newInstance(10240, 1); + Resource amResource6 = Resource.newInstance(RM_SCHEDULER_MAXIMUM_ALLOCATION_MB_VALUE, 1); Resource amResource7 = Resource.newInstance(1024, 1); Resource amResource8 = Resource.newInstance(1024, 1); int amPriority = RMAppAttemptImpl.AM_CONTAINER_PRIORITY.getPriority(); @@ -4159,7 +4253,7 @@ public void testQueueMaxAMShareWithContainerReservation() throws Exception { ApplicationAttemptId attId3 = createAppAttemptId(3, 1); createApplicationWithAMResource(attId3, "queue1", "user1", amResource3); - createSchedulingRequestExistingApplication(10240, 1, amPriority, attId3); + createSchedulingRequestExistingApplication(RM_SCHEDULER_MAXIMUM_ALLOCATION_MB_VALUE, 1, amPriority, attId3); FSAppAttempt app3 = scheduler.getSchedulerApp(attId3); scheduler.update(); // app3 reserves a container on node1 because node1's available resource @@ -4233,7 +4327,7 @@ public void testQueueMaxAMShareWithContainerReservation() throws Exception { ApplicationAttemptId attId6 = createAppAttemptId(6, 1); createApplicationWithAMResource(attId6, "queue1", "user1", amResource6); - createSchedulingRequestExistingApplication(10240, 1, amPriority, attId6); + createSchedulingRequestExistingApplication(RM_SCHEDULER_MAXIMUM_ALLOCATION_MB_VALUE, 1, amPriority, attId6); FSAppAttempt app6 = scheduler.getSchedulerApp(attId6); scheduler.update(); // app6 can't reserve a container on node1 because @@ -4322,7 +4416,7 @@ public void testQueueMaxAMShareWithContainerReservation() throws Exception { // app6 turns the reservation into an allocation on node2. scheduler.handle(updateE2); assertEquals("Application6's AM requests 10240 MB memory", - 10240, app6.getAMResource().getMemorySize()); + RM_SCHEDULER_MAXIMUM_ALLOCATION_MB_VALUE, app6.getAMResource().getMemorySize()); assertEquals("Application6's AM should be running", 1, app6.getLiveContainers().size()); assertEquals("Queue1's AM resource usage should be 11264 MB memory", diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/allocationfile/AllocationFileQueue.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/allocationfile/AllocationFileQueue.java index f1afe6979fc..62b21874f6d 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/allocationfile/AllocationFileQueue.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/allocationfile/AllocationFileQueue.java @@ -60,9 +60,13 @@ String render() { () -> AllocationFileWriter .createNumberSupplier(properties.getFairSharePreemptionTimeout())); AllocationFileWriter.addIfPresent(pw, "fairSharePreemptionThreshold", - () -> AllocationFileWriter - .createNumberSupplier( - properties.getFairSharePreemptionThreshold())); + () -> AllocationFileWriter + .createNumberSupplier( + properties.getFairSharePreemptionThreshold())); + AllocationFileWriter.addIfPresent(pw, "maxContainerResources", + () -> AllocationFileWriter + .createNumberSupplier( + properties.getMaxContainerResources())); printEndTag(pw); pw.close(); return sw.toString(); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/allocationfile/AllocationFileQueueBuilder.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/allocationfile/AllocationFileQueueBuilder.java index a2faf1da318..3f4d029b5c8 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/allocationfile/AllocationFileQueueBuilder.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/allocationfile/AllocationFileQueueBuilder.java @@ -94,6 +94,11 @@ public AllocationFileQueueBuilder fairSharePreemptionThreshold( return this; } + public AllocationFileQueueBuilder maxContainerResources(String maxContainerResources){ + this.queuePropertiesBuilder.maxContainerResources(maxContainerResources); + return this; + } + public AllocationFileQueueBuilder subQueue(String queueName) { if (this instanceof AllocationFileSimpleQueueBuilder) { return new AllocationFileSubQueueBuilder( diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/allocationfile/AllocationFileQueueProperties.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/allocationfile/AllocationFileQueueProperties.java index 2c01144a152..b923cca79c3 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/allocationfile/AllocationFileQueueProperties.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/allocationfile/AllocationFileQueueProperties.java @@ -33,6 +33,7 @@ private final String maxChildResources; private final Integer fairSharePreemptionTimeout; private final Double fairSharePreemptionThreshold; + private final String maxContainerResources; AllocationFileQueueProperties(Builder builder) { this.queueName = builder.queueName; @@ -48,6 +49,7 @@ this.maxChildResources = builder.maxChildResources; this.fairSharePreemptionTimeout = builder.fairSharePreemptionTimeout; this.fairSharePreemptionThreshold = builder.fairSharePreemptionThreshold; + this.maxContainerResources = builder.maxContainerResources; } public String getQueueName() { @@ -102,6 +104,8 @@ public Double getFairSharePreemptionThreshold() { return fairSharePreemptionThreshold; } + public String getMaxContainerResources() { return maxContainerResources; } + /** * Builder class for {@link AllocationFileQueueProperties}. */ @@ -119,6 +123,7 @@ public Double getFairSharePreemptionThreshold() { private String maxChildResources; private Integer fairSharePreemptionTimeout; private Double fairSharePreemptionThreshold; + private String maxContainerResources; Builder() { } @@ -167,6 +172,11 @@ public Builder maxAMShare(Double maxAMShare) { return this; } + public Builder maxContainerResources(String maxContainerResources) { + this.maxContainerResources = maxContainerResources; + return this; + } + public Builder minSharePreemptionTimeout( Integer minSharePreemptionTimeout) { this.minSharePreemptionTimeout = minSharePreemptionTimeout; diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/allocationfile/TestQueueMaxContainerAllocationValidator.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/allocationfile/TestQueueMaxContainerAllocationValidator.java new file mode 100644 index 00000000000..c8491814fc6 --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/allocationfile/TestQueueMaxContainerAllocationValidator.java @@ -0,0 +1,119 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * http://www.apache.org/licenses/LICENSE-2.0 + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.allocationfile; + +import org.apache.hadoop.yarn.api.records.Resource; +import org.apache.hadoop.yarn.conf.YarnConfiguration; +import org.apache.hadoop.yarn.exceptions.YarnRuntimeException; +import org.apache.hadoop.yarn.server.resourcemanager.scheduler.ResourceScheduler; +import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.*; +import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.allocation.QueueMaxContainerAllocationValidator; +import org.junit.Assert; +import org.junit.Before; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.mockito.Mock; +import org.mockito.Mockito; +import org.mockito.runners.MockitoJUnitRunner; + +import java.io.FileWriter; +import java.io.IOException; +import java.io.PrintWriter; +import java.util.Arrays; + +import static org.apache.hadoop.yarn.conf.YarnConfiguration.RM_SCHEDULER_MAXIMUM_ALLOCATION_MB; +import static org.apache.hadoop.yarn.conf.YarnConfiguration.RM_SCHEDULER_MAXIMUM_ALLOCATION_VCORES; + +@RunWith(MockitoJUnitRunner.class) +public class TestQueueMaxContainerAllocationValidator { + + private QueueMaxContainerAllocationValidator target; + + @Mock + private QueueManager queueManager; + + private YarnConfiguration configuration = new YarnConfiguration(); + private FairScheduler fairScheduler; + private int maxVCores = 2; + private int maxMemory = 1000; + private FSParentQueue rootQueue; + private FSLeafQueue leafQueue; + + @Before + public void setup() { + fairScheduler = new FairScheduler(); + fairScheduler.init(configuration); + configuration.setInt( + YarnConfiguration.RM_SCHEDULER_MAXIMUM_ALLOCATION_VCORES, maxVCores); + configuration.setInt(YarnConfiguration.RM_SCHEDULER_MAXIMUM_ALLOCATION_MB, + maxMemory); + rootQueue = new FSParentQueue("root", fairScheduler, null); + leafQueue = new FSLeafQueue("root.leaf", fairScheduler, rootQueue); + target = + new QueueMaxContainerAllocationValidator(queueManager, configuration); + } + + @Test(expected = YarnRuntimeException.class) + public void validateRootQueueMaxContainerAllocationTest() throws IOException { + FSParentQueue rootQueue = new FSParentQueue("root", fairScheduler, null); + rootQueue.setMaxContainerResources(Resource.newInstance(10, 1)); + Mockito.when(queueManager.getQueue("root")).thenReturn(rootQueue); + target.validate(); + } + + @Test + public void validContainerAllocationTest() throws IOException { + mockQueues(maxMemory, maxVCores); + target.validate(); + // No exception expected + } + + @Test + public void tooHighMemoryMaxContainerAllocationTest() throws IOException { + mockQueues(maxMemory + 1, maxVCores); + try { + target.validate(); + Assert.fail("Should throw exception!"); + } catch (YarnRuntimeException e) { + Assert.assertEquals( + QueueMaxContainerAllocationValidator.createExceptionText( + RM_SCHEDULER_MAXIMUM_ALLOCATION_MB, leafQueue, maxMemory), + e.getMessage()); + } + } + + @Test + public void tooManyVCoresMaxContainerAllocationTest() throws IOException { + mockQueues(maxMemory, maxVCores + 1); + try { + target.validate(); + Assert.fail("Should throw exception!"); + } catch (YarnRuntimeException e) { + Assert.assertEquals( + QueueMaxContainerAllocationValidator.createExceptionText( + RM_SCHEDULER_MAXIMUM_ALLOCATION_VCORES, leafQueue, maxVCores), + e.getMessage()); + } + } + + private void mockQueues(int memory, int vCores) { + leafQueue.setMaxContainerResources(Resource.newInstance(memory, vCores)); + Mockito.when(queueManager.getQueues()) + .thenReturn(Arrays.asList(rootQueue, leafQueue)); + Mockito.when(queueManager.getQueue("root")).thenReturn(rootQueue); + } +} diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/FairScheduler.md b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/FairScheduler.md index b5bcbf5c8e9..d246b85b6d8 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/FairScheduler.md +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/FairScheduler.md @@ -90,6 +90,8 @@ The allocation file must be in XML format. The format contains five types of ele * **maxResources**: maximum resources a queue will allocated, expressed in the form of "X%", "X% cpu, Y% memory", "X mb, Y vcores", or "vcores=X, memory-mb=Y". The last form is required when specifying resources other than memory and CPU. In the last form, X and Y can either be a percentage or an integer resource value without units. In the latter case the units will be inferred from the default units configured for that resource. A queue will not be assigned a container that would put its aggregate usage over this limit. + * **maxContainerResources**: maximum resources a queue can allocate for a single container, expressed in the form of "X mb, Y vcores" or "vcores=X, memory-mb=Y". The latter form is required when specifying resources other than memory and CPU. If the property is not set it's value is inherited from a parent queue. It's default value is **yarn.scheduler.maximum-allocation-mb**. Cannot be higher than **maxResources**. This property is invalid for root queue. + * **maxChildResources**: maximum resources an ad hoc child queue will allocated, expressed in the form of "X%", "X% cpu, Y% memory", "X mb, Y vcores", or "vcores=X, memory-mb=Y". The last form is required when specifying resources other than memory and CPU. In the last form, X and Y can either be a percentage or an integer resource value without units. In the latter case the units will be inferred from the default units configured for that resource. An ad hoc child queue will not be assigned a container that would put its aggregate usage over this limit. * **maxRunningApps**: limit the number of apps from the queue to run at once