diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/AbstractCSQueue.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/AbstractCSQueue.java index 5c520b783b9..d3c435b4df4 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/AbstractCSQueue.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/AbstractCSQueue.java @@ -18,7 +18,6 @@ package org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity; -import org.apache.commons.lang3.StringUtils; import org.apache.hadoop.classification.InterfaceAudience.Private; import org.apache.hadoop.ipc.Server; import org.apache.hadoop.security.AccessControlException; @@ -37,7 +36,6 @@ import org.apache.hadoop.yarn.api.records.Resource; import org.apache.hadoop.yarn.api.records.ResourceInformation; import org.apache.hadoop.yarn.exceptions.YarnException; -import org.apache.hadoop.yarn.exceptions.YarnRuntimeException; import org.apache.hadoop.yarn.factories.RecordFactory; import org.apache.hadoop.yarn.factory.providers.RecordFactoryProvider; import org.apache.hadoop.yarn.security.AccessRequest; @@ -77,30 +75,25 @@ import java.util.concurrent.locks.ReentrantReadWriteLock; import static org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacitySchedulerConfiguration.DOT; -import static org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacitySchedulerConfiguration.ROOT; -import static org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacitySchedulerConfiguration.UNDEFINED; public abstract class AbstractCSQueue implements CSQueue { - private static final Logger LOG = LoggerFactory.getLogger(AbstractCSQueue.class); - volatile CSQueue parent; - final String queueName; - private final String queuePath; + protected final QueueAllocationSettings queueAllocationSettings; + protected final BasicQueueProperties basicQueueProperties; + protected QueueNodeLabelsSettings queueNodeLabelsSettings; + private volatile QueueAppLifetimeAndLimitSettings queueAppLifetimeSettings; + private CSQueuePreemptionSettings preemptionSettings; + volatile int numContainers; - final Resource minimumAllocation; - volatile Resource maximumAllocation; private volatile QueueState state = null; final CSQueueMetrics metrics; protected final PrivilegedEntity queueEntity; final ResourceCalculator resourceCalculator; - Set accessibleLabels; - protected Set configuredNodeLabels; Set resourceTypes; final RMNodeLabelsManager labelManager; - String defaultLabelExpression; private String multiNodeSortingPolicyName = null; Map acls = @@ -114,19 +107,8 @@ // used-capacity/abs-used-capacity/capacity/abs-capacity, // etc. QueueCapacities queueCapacities; - QueueResourceQuotas queueResourceQuotas; - // -1 indicates lifetime is disabled - private volatile long maxApplicationLifetime = -1; - - private volatile long defaultApplicationLifetime = -1; - - // Indicates if this queue's default lifetime was set by a config property, - // either at this level or anywhere in the queue's hierarchy. - private volatile boolean defaultAppLifetimeWasSpecifiedInConfig = false; - private CSQueuePreemption preemptionSettings; - public enum CapacityConfigType { // FIXME, from what I can see, Percentage mode can almost apply to weighted // and percentage mode at the same time, there's only small area need to be @@ -148,7 +130,6 @@ volatile Priority priority = Priority.newInstance(0); private UserWeights userWeights = UserWeights.createEmpty(); - private int maxParallelApps; // is it a dynamic queue? private boolean dynamicQueue = false; @@ -165,12 +146,8 @@ public AbstractCSQueue(CapacitySchedulerContext cs, public AbstractCSQueue(CapacitySchedulerContext cs, CapacitySchedulerConfiguration configuration, String queueName, CSQueue parent, CSQueue old) { - this.labelManager = cs.getRMContext().getNodeLabelManager(); - this.parent = parent; - this.queueName = queueName; - this.queuePath = ((parent == null) ? "" : (parent.getQueuePath() + ".")) - + this.queueName; + this.basicQueueProperties = new BasicQueueProperties(parent, queueName); this.resourceCalculator = cs.getResourceCalculator(); this.activitiesManager = cs.getActivitiesManager(); @@ -181,7 +158,7 @@ public AbstractCSQueue(CapacitySchedulerContext cs, cs.getConfiguration().getEnableUserMetrics(), configuration); this.csContext = cs; - this.minimumAllocation = csContext.getMinimumResourceCapability(); + this.queueAllocationSettings = new QueueAllocationSettings(csContext); // initialize ResourceUsage queueUsage = new ResourceUsage(); @@ -206,12 +183,12 @@ protected void setupConfigurableCapacities() { protected void setupConfigurableCapacities( CapacitySchedulerConfiguration configuration) { CSQueueUtils.loadCapacitiesByLabelsFromConf(getQueuePath(), queueCapacities, - configuration, configuredNodeLabels); + configuration, this.queueNodeLabelsSettings.getConfiguredNodeLabels()); } @Override public String getQueuePath() { - return queuePath; + return basicQueueProperties.getQueuePath(); } @Override @@ -265,31 +242,31 @@ public CSQueueMetrics getMetrics() { @Override public String getQueueShortName() { - return queueName; + return basicQueueProperties.getQueueName(); } @Override public String getQueueName() { - return queueName; + return basicQueueProperties.getQueueName(); } @Override - public PrivilegedEntity getPrivilegedEntity() { - return queueEntity; + public CSQueue getParent() { + return basicQueueProperties.getParent(); } @Override - public CSQueue getParent() { - return parent; + public void setParent(CSQueue newParentQueue) { + this.basicQueueProperties.setParent(newParentQueue); } @Override - public void setParent(CSQueue newParentQueue) { - this.parent = newParentQueue; + public PrivilegedEntity getPrivilegedEntity() { + return queueEntity; } public Set getAccessibleNodeLabels() { - return accessibleLabels; + return queueNodeLabelsSettings.getAccessibleNodeLabels(); } @Override @@ -310,7 +287,7 @@ void setMaxCapacity(float maximumCapacity) { CSQueueUtils.checkMaxCapacity(getQueuePath(), queueCapacities.getCapacity(), maximumCapacity); float absMaxCapacity = CSQueueUtils.computeAbsoluteMaximumCapacity( - maximumCapacity, parent); + maximumCapacity, getParent()); CSQueueUtils.checkAbsoluteCapacity(getQueuePath(), queueCapacities.getAbsoluteCapacity(), absMaxCapacity); @@ -332,7 +309,7 @@ void setMaxCapacity(String nodeLabel, float maximumCapacity) { CSQueueUtils.checkMaxCapacity(getQueuePath(), queueCapacities.getCapacity(nodeLabel), maximumCapacity); float absMaxCapacity = CSQueueUtils.computeAbsoluteMaximumCapacity( - maximumCapacity, parent); + maximumCapacity, getParent()); CSQueueUtils.checkAbsoluteCapacity(getQueuePath(), queueCapacities.getAbsoluteCapacity(nodeLabel), absMaxCapacity); @@ -346,7 +323,7 @@ void setMaxCapacity(String nodeLabel, float maximumCapacity) { @Override public String getDefaultNodeLabelExpression() { - return defaultLabelExpression; + return this.queueNodeLabelsSettings.getDefaultLabelExpression(); } protected void setupQueueConfigs(Resource clusterResource, @@ -360,7 +337,8 @@ protected void setupQueueConfigs(Resource clusterResource, } // Collect and set the Node label configuration - initializeNodeLabels(configuration); + this.queueNodeLabelsSettings = new QueueNodeLabelsSettings(configuration, getParent(), + getQueuePath(), csContext); // Initialize the queue capacities setupConfigurableCapacities(configuration); @@ -378,7 +356,8 @@ protected void setupQueueConfigs(Resource clusterResource, // Setup queue's maximumAllocation respecting the global // and the queue settings - setupMaximumAllocation(configuration); + this.queueAllocationSettings.setupMaximumAllocation(configuration, getQueuePath(), + getParent(), csContext); // Initialize the queue state based on previous state, configured state // and its parent state @@ -398,7 +377,7 @@ protected void setupQueueConfigs(Resource clusterResource, this, labelManager, null); // Store preemption settings - this.preemptionSettings = new CSQueuePreemption(this, csContext, configuration); + this.preemptionSettings = new CSQueuePreemptionSettings(this, csContext, configuration); this.priority = configuration.getQueuePriority( getQueuePath()); @@ -407,7 +386,8 @@ protected void setupQueueConfigs(Resource clusterResource, configuration.getMultiNodesSortingAlgorithmPolicy(getQueuePath())); // Setup application related limits - setupApplicationLimits(configuration); + this.queueAppLifetimeSettings = new QueueAppLifetimeAndLimitSettings(configuration, + this, getQueuePath()); } finally { writeLock.unlock(); } @@ -434,135 +414,7 @@ protected void setDynamicQueueProperties( if (parentNodeLabels != null && parentNodeLabels.size() > 1) { csContext.getCapacitySchedulerQueueManager().getConfiguredNodeLabels() - .setLabelsByQueue(queuePath, new HashSet<>(parentNodeLabels)); - } - } - } - - private void initializeNodeLabels( - CapacitySchedulerConfiguration configuration) throws IOException { - // Collect and store labels - this.accessibleLabels = - configuration.getAccessibleNodeLabels(getQueuePath()); - this.defaultLabelExpression = - configuration.getDefaultNodeLabelExpression( - getQueuePath()); - - // Inherit labels from parent if not set - if (this.accessibleLabels == null && parent != null) { - this.accessibleLabels = parent.getAccessibleNodeLabels(); - } - - // If the accessible labels is not null and the queue has a parent with a - // similar set of labels copy the defaultNodeLabelExpression from the parent - if (this.accessibleLabels != null && parent != null - && this.defaultLabelExpression == null && - this.accessibleLabels.containsAll(parent.getAccessibleNodeLabels())) { - this.defaultLabelExpression = parent.getDefaultNodeLabelExpression(); - } - - if (csContext.getCapacitySchedulerQueueManager() != null - && csContext.getCapacitySchedulerQueueManager() - .getConfiguredNodeLabels() != null) { - if (getQueuePath().equals(ROOT)) { - this.configuredNodeLabels = csContext.getCapacitySchedulerQueueManager() - .getConfiguredNodeLabels().getAllConfiguredLabels(); - } else { - this.configuredNodeLabels = csContext.getCapacitySchedulerQueueManager() - .getConfiguredNodeLabels().getLabelsByQueue(getQueuePath()); - } - } else { - // Fallback to suboptimal but correct logic - this.configuredNodeLabels = csContext.getConfiguration() - .getConfiguredNodeLabels(queuePath); - } - - // Validate the initialized settings - validateNodeLabels(); - } - - private void validateNodeLabels() throws IOException { - // Check if labels of this queue is a subset of parent queue, only do this - // when the queue in question is not root - if (parent != null && parent.getParent() != null) { - if (parent.getAccessibleNodeLabels() != null && !parent - .getAccessibleNodeLabels().contains(RMNodeLabelsManager.ANY)) { - // if parent isn't "*", child shouldn't be "*" too - if (this.getAccessibleNodeLabels().contains( - RMNodeLabelsManager.ANY)) { - throw new IOException("Parent's accessible queue is not ANY(*), " - + "but child's accessible queue is *"); - } else{ - Set diff = Sets.difference(this.getAccessibleNodeLabels(), - parent.getAccessibleNodeLabels()); - if (!diff.isEmpty()) { - throw new IOException( - "Some labels of child queue is not a subset " - + "of parent queue, these labels=[" + StringUtils - .join(diff, ",") + "]"); - } - } - } - } - } - - private void setupApplicationLimits(CapacitySchedulerConfiguration configuration) { - // Store max parallel apps property - this.maxParallelApps = configuration.getMaxParallelAppsForQueue(getQueuePath()); - - maxApplicationLifetime = getInheritedMaxAppLifetime(this, configuration); - defaultApplicationLifetime = - getInheritedDefaultAppLifetime(this, configuration, - maxApplicationLifetime); - } - - private void setupMaximumAllocation(CapacitySchedulerConfiguration csConf) { - String myQueuePath = getQueuePath(); - /* YARN-10869: When using AutoCreatedLeafQueues, the passed configuration - * object is a cloned one containing only the template configs - * (see ManagedParentQueue#getLeafQueueConfigs). To ensure that the actual - * cluster maximum allocation is fetched the original config object should - * be used. - */ - Resource clusterMax = ResourceUtils - .fetchMaximumAllocationFromConfig(this.csContext.getConfiguration()); - Resource queueMax = csConf.getQueueMaximumAllocation(myQueuePath); - - maximumAllocation = Resources.clone( - parent == null ? clusterMax : parent.getMaximumAllocation()); - - String errMsg = - "Queue maximum allocation cannot be larger than the cluster setting" - + " for queue " + myQueuePath - + " max allocation per queue: %s" - + " cluster setting: " + clusterMax; - - if (queueMax == Resources.none()) { - // Handle backward compatibility - long queueMemory = csConf.getQueueMaximumAllocationMb(myQueuePath); - int queueVcores = csConf.getQueueMaximumAllocationVcores(myQueuePath); - if (queueMemory != UNDEFINED) { - maximumAllocation.setMemorySize(queueMemory); - } - - if (queueVcores != UNDEFINED) { - maximumAllocation.setVirtualCores(queueVcores); - } - - if ((queueMemory != UNDEFINED && queueMemory > clusterMax.getMemorySize() - || (queueVcores != UNDEFINED - && queueVcores > clusterMax.getVirtualCores()))) { - throw new IllegalArgumentException( - String.format(errMsg, maximumAllocation)); - } - } else { - // Queue level maximum-allocation can't be larger than cluster setting - for (ResourceInformation ri : queueMax.getResources()) { - if (ri.compareTo(clusterMax.getResourceInformation(ri.getName())) > 0) { - throw new IllegalArgumentException(String.format(errMsg, queueMax)); - } - - maximumAllocation.setResourceInformation(ri.getName(), ri); + .setLabelsByQueue(getQueuePath(), new HashSet<>(parentNodeLabels)); } } } @@ -602,12 +454,12 @@ protected boolean checkConfigTypeIsAbsoluteResource(String queuePath, protected void updateCapacityConfigType() { this.capacityConfigType = CapacityConfigType.NONE; - for (String label : configuredNodeLabels) { + for (String label : queueNodeLabelsSettings.getConfiguredNodeLabels()) { LOG.debug("capacityConfigType is '{}' for queue {}", capacityConfigType, getQueuePath()); CapacityConfigType localType = checkConfigTypeIsAbsoluteResource( - queuePath, label) ? CapacityConfigType.ABSOLUTE_RESOURCE + getQueuePath(), label) ? CapacityConfigType.ABSOLUTE_RESOURCE : CapacityConfigType.PERCENTAGE; if (this.capacityConfigType.equals(CapacityConfigType.NONE)) { @@ -621,12 +473,13 @@ protected void updateCapacityConfigType() { } protected void updateConfigurableResourceLimits(Resource clusterResource) { - for (String label : configuredNodeLabels) { - final Resource minResource = getMinimumAbsoluteResource(queuePath, label); - Resource maxResource = getMaximumAbsoluteResource(queuePath, label); + for (String label : queueNodeLabelsSettings.getConfiguredNodeLabels()) { + final Resource minResource = getMinimumAbsoluteResource(getQueuePath(), label); + Resource maxResource = getMaximumAbsoluteResource(getQueuePath(), label); - if (parent != null) { - final Resource parentMax = parent.getQueueResourceQuotas().getConfiguredMaxResource(label); + if (getParent() != null) { + final Resource parentMax = getParent().getQueueResourceQuotas() + .getConfiguredMaxResource(label); validateMinResourceIsNotGreaterThanMaxResource(maxResource, parentMax, clusterResource, "Max resource configuration " + maxResource + " is greater than parents max value:" @@ -667,7 +520,7 @@ private void validateMinResourceIsNotGreaterThanMaxResource(Resource minResource private void validateAbsoluteVsPercentageCapacityConfig( CapacityConfigType localType) { - if (!queuePath.equals("root") + if (!getQueuePath().equals("root") && !this.capacityConfigType.equals(localType)) { throw new IllegalArgumentException("Queue '" + getQueuePath() + "' should use either percentage based capacity" @@ -690,7 +543,7 @@ public Resource getEffectiveCapacity(String label) { public Resource getEffectiveCapacityDown(String label, Resource factor) { return Resources.normalizeDown(resourceCalculator, getQueueResourceQuotas().getEffectiveMinResource(label), - minimumAllocation); + queueAllocationSettings.getMinimumAllocation()); } @Override @@ -703,14 +556,14 @@ public Resource getEffectiveMaxCapacity(String label) { public Resource getEffectiveMaxCapacityDown(String label, Resource factor) { return Resources.normalizeDown(resourceCalculator, getQueueResourceQuotas().getEffectiveMaxResource(label), - minimumAllocation); + queueAllocationSettings.getMinimumAllocation()); } private void initializeQueueState(CapacitySchedulerConfiguration configuration) { QueueState previousState = getState(); QueueState configuredState = configuration .getConfiguredState(getQueuePath()); - QueueState parentState = (parent == null) ? null : parent.getState(); + QueueState parentState = (getParent() == null) ? null : getParent().getState(); // verify that we can not any value for State other than RUNNING/STOPPED if (configuredState != null && configuredState != QueueState.RUNNING @@ -735,8 +588,8 @@ private void initializeQueueState(CapacitySchedulerConfiguration configuration) } else if (configuredState == QueueState.RUNNING && parentState != QueueState.RUNNING) { throw new IllegalArgumentException( - "The parent queue:" + parent.getQueuePath() - + " cannot be STOPPED as the child queue:" + queuePath + "The parent queue:" + getParent().getQueuePath() + + " cannot be STOPPED as the child queue:" + getQueuePath() + " is in RUNNING state."); } else { updateQueueState(configuredState); @@ -766,21 +619,21 @@ protected QueueInfo getQueueInfo() { // consistency here. // TODO, improve this QueueInfo queueInfo = recordFactory.newRecordInstance(QueueInfo.class); - queueInfo.setQueueName(queueName); - queueInfo.setQueuePath(queuePath); - queueInfo.setAccessibleNodeLabels(accessibleLabels); + queueInfo.setQueueName(getQueueName()); + queueInfo.setQueuePath(getQueuePath()); + queueInfo.setAccessibleNodeLabels(queueNodeLabelsSettings.getAccessibleNodeLabels()); queueInfo.setCapacity(queueCapacities.getCapacity()); queueInfo.setMaximumCapacity(queueCapacities.getMaximumCapacity()); queueInfo.setQueueState(getState()); - queueInfo.setDefaultNodeLabelExpression(defaultLabelExpression); + queueInfo.setDefaultNodeLabelExpression(queueNodeLabelsSettings.getDefaultLabelExpression()); queueInfo.setCurrentCapacity(getUsedCapacity()); queueInfo.setQueueStatistics(getQueueStatistics()); - queueInfo.setPreemptionDisabled(getPreemptionDisabled()); + queueInfo.setPreemptionDisabled(preemptionSettings.isPreemptionDisabled()); queueInfo.setIntraQueuePreemptionDisabled( getIntraQueuePreemptionDisabled()); queueInfo.setQueueConfigurations(getQueueConfigurations()); queueInfo.setWeight(queueCapacities.getWeight()); - queueInfo.setMaxParallelApps(maxParallelApps); + queueInfo.setMaxParallelApps(queueAppLifetimeSettings.getMaxParallelApps()); return queueInfo; } @@ -845,12 +698,12 @@ public QueueStatistics getQueueStatistics() { @Private public Resource getMaximumAllocation() { - return maximumAllocation; + return queueAllocationSettings.getMaximumAllocation(); } @Private public Resource getMinimumAllocation() { - return minimumAllocation; + return queueAllocationSettings.getMinimumAllocation(); } void allocateResource(Resource clusterResource, @@ -905,8 +758,7 @@ public boolean getPreemptionDisabled() { @Private public boolean getIntraQueuePreemptionDisabled() { - return preemptionSettings.isIntraQueuePreemptionDisabledInHierarchy() || - preemptionSettings.isPreemptionDisabled(); + return preemptionSettings.getIntraQueuePreemptionDisabled(); } @Private @@ -934,76 +786,6 @@ public QueueResourceQuotas getQueueResourceQuotas() { return readLock; } - private long getInheritedMaxAppLifetime(CSQueue q, - CapacitySchedulerConfiguration conf) { - CSQueue parentQ = q.getParent(); - long maxAppLifetime = conf.getMaximumLifetimePerQueue(q.getQueuePath()); - - // If q is the root queue, then get max app lifetime from conf. - if (parentQ == null) { - return maxAppLifetime; - } - - // If this is not the root queue, get this queue's max app lifetime - // from the conf. The parent's max app lifetime will be used if it's - // not set for this queue. - // A value of 0 will override the parent's value and means no max lifetime. - // A negative value means that the parent's max should be used. - long parentsMaxAppLifetime = getParent().getMaximumApplicationLifetime(); - return (maxAppLifetime >= 0) ? maxAppLifetime : parentsMaxAppLifetime; - } - - private long getInheritedDefaultAppLifetime(CSQueue q, - CapacitySchedulerConfiguration conf, long myMaxAppLifetime) { - CSQueue parentQ = q.getParent(); - long defaultAppLifetime = conf.getDefaultLifetimePerQueue(getQueuePath()); - defaultAppLifetimeWasSpecifiedInConfig = - (defaultAppLifetime >= 0 - || (parentQ != null && - parentQ.getDefaultAppLifetimeWasSpecifiedInConfig())); - - // If q is the root queue, then get default app lifetime from conf. - if (parentQ == null) { - return defaultAppLifetime; - } - - // If this is not the root queue, get the parent's default app lifetime. The - // parent's default app lifetime will be used if not set for this queue. - long parentsDefaultAppLifetime = - getParent().getDefaultApplicationLifetime(); - - // Negative value indicates default lifetime was not set at this level. - // If default lifetime was not set at this level, calculate it based on - // parent's default lifetime or current queue's max lifetime. - if (defaultAppLifetime < 0) { - // If default lifetime was not set at this level but was set somewhere in - // the parent's hierarchy, set default lifetime to parent queue's default - // only if parent queue's lifetime is less than current queue's max - // lifetime. Otherwise, use current queue's max lifetime value for its - // default lifetime. - if (defaultAppLifetimeWasSpecifiedInConfig) { - defaultAppLifetime = - Math.min(parentsDefaultAppLifetime, myMaxAppLifetime); - } else { - // Default app lifetime value was not set anywhere in this queue's - // hierarchy. Use current queue's max lifetime as its default. - defaultAppLifetime = myMaxAppLifetime; - } - } // else if >= 0, default lifetime was set at this level. Just use it. - - if (myMaxAppLifetime > 0 && - defaultAppLifetime > myMaxAppLifetime) { - throw new YarnRuntimeException( - "Default lifetime " + defaultAppLifetime - + " can't exceed maximum lifetime " + myMaxAppLifetime); - } - - if (defaultAppLifetime <= 0) { - defaultAppLifetime = myMaxAppLifetime; - } - return defaultAppLifetime; - } - private Resource getCurrentLimitResource(String nodePartition, Resource clusterResource, ResourceLimits currentResourceLimits, SchedulingMode schedulingMode) { @@ -1163,39 +945,39 @@ private void countAndUpdate(String partition, Resource resource, @Override public void incReservedResource(String partition, Resource reservedRes) { count(partition, reservedRes, queueUsage::incReserved, - parent == null ? null : parent::incReservedResource); + getParent() == null ? null : getParent()::incReservedResource); } @Override public void decReservedResource(String partition, Resource reservedRes) { count(partition, reservedRes, queueUsage::decReserved, - parent == null ? null : parent::decReservedResource); + getParent() == null ? null : getParent()::decReservedResource); } @Override public void incPendingResource(String nodeLabel, Resource resourceToInc) { count(nodeLabel, resourceToInc, queueUsage::incPending, - parent == null ? null : parent::incPendingResource); + getParent() == null ? null : getParent()::incPendingResource); } @Override public void decPendingResource(String nodeLabel, Resource resourceToDec) { count(nodeLabel, resourceToDec, queueUsage::decPending, - parent == null ? null : parent::decPendingResource); + getParent() == null ? null : getParent()::decPendingResource); } @Override public void incUsedResource(String nodeLabel, Resource resourceToInc, SchedulerApplicationAttempt application) { countAndUpdate(nodeLabel, resourceToInc, queueUsage::incUsed, - parent == null ? null : parent::incUsedResource); + getParent() == null ? null : getParent()::incUsedResource); } @Override public void decUsedResource(String nodeLabel, Resource resourceToDec, SchedulerApplicationAttempt application) { countAndUpdate(nodeLabel, resourceToDec, queueUsage::decUsed, - parent == null ? null : parent::decUsedResource); + getParent() == null ? null : getParent()::decUsedResource); } /** @@ -1208,25 +990,6 @@ boolean hasPendingResourceRequest(String nodePartition, queueUsage, nodePartition, cluster, schedulingMode); } - public boolean accessibleToPartition(String nodePartition) { - // if queue's label is *, it can access any node - if (accessibleLabels != null - && accessibleLabels.contains(RMNodeLabelsManager.ANY)) { - return true; - } - // any queue can access to a node without label - if (nodePartition == null - || nodePartition.equals(RMNodeLabelsManager.NO_LABEL)) { - return true; - } - // a queue can access to a node only if it contains any label of the node - if (accessibleLabels != null && accessibleLabels.contains(nodePartition)) { - return true; - } - // sorry, you cannot access - return false; - } - @Override public Priority getDefaultApplicationPriority() { // TODO add dummy implementation @@ -1322,8 +1085,8 @@ public boolean accept(Resource cluster, } - if (parent != null && checkParentQueue) { - return parent.accept(cluster, request); + if (getParent() != null && checkParentQueue) { + return getParent().accept(cluster, request); } return true; @@ -1409,32 +1172,32 @@ public void setMultiNodeSortingPolicyName(String policyName) { } public long getMaximumApplicationLifetime() { - return maxApplicationLifetime; + return queueAppLifetimeSettings.getMaxApplicationLifetime(); } public long getDefaultApplicationLifetime() { - return defaultApplicationLifetime; + return queueAppLifetimeSettings.getDefaultApplicationLifetime(); } public boolean getDefaultAppLifetimeWasSpecifiedInConfig() { - return defaultAppLifetimeWasSpecifiedInConfig; + return queueAppLifetimeSettings.isDefaultAppLifetimeWasSpecifiedInConfig(); } public void setMaxParallelApps(int maxParallelApps) { - this.maxParallelApps = maxParallelApps; + this.queueAppLifetimeSettings.setMaxParallelApps(maxParallelApps); } @Override public int getMaxParallelApps() { - return maxParallelApps; + return this.queueAppLifetimeSettings.getMaxParallelApps(); } abstract int getNumRunnableApps(); protected void updateAbsoluteCapacities() { QueueCapacities parentQueueCapacities = null; - if (parent != null) { - parentQueueCapacities = parent.getQueueCapacities(); + if (getParent() != null) { + parentQueueCapacities = getParent().getQueueCapacities(); } CSQueueUtils.updateAbsoluteCapacitiesByNodeLabels(queueCapacities, @@ -1454,7 +1217,7 @@ private Resource createNormalizedMinResource(Resource minResource, ret.setResourceValue(i, (long) (nResourceInformation.getValue() * ratio)); if (LOG.isDebugEnabled()) { - LOG.debug("Updating min resource for Queue: " + queuePath + " as " + ret + LOG.debug("Updating min resource for Queue: " + getQueuePath() + " as " + ret .getResourceInformation(i) + ", Actual resource: " + nResourceInformation.getValue() + ", ratio: " + ratio); } @@ -1465,10 +1228,10 @@ private Resource createNormalizedMinResource(Resource minResource, private Resource getOrInheritMaxResource(Resource resourceByLabel, String label) { Resource parentMaxResource = - parent.getQueueResourceQuotas().getConfiguredMaxResource(label); + getParent().getQueueResourceQuotas().getConfiguredMaxResource(label); if (parentMaxResource.equals(Resources.none())) { parentMaxResource = - parent.getQueueResourceQuotas().getEffectiveMaxResource(label); + getParent().getQueueResourceQuotas().getEffectiveMaxResource(label); } Resource configuredMaxResource = @@ -1483,12 +1246,12 @@ private Resource getOrInheritMaxResource(Resource resourceByLabel, String label) void updateMaxAppRelatedField(CapacitySchedulerConfiguration conf, LeafQueue leafQueue) { - int maxApplications = conf.getMaximumApplicationsPerQueue(queuePath); + int maxApplications = conf.getMaximumApplicationsPerQueue(getQueuePath()); int maxGlobalPerQueueApps = conf.getGlobalMaximumApplicationsPerQueue(); String maxLabel = RMNodeLabelsManager.NO_LABEL; if (maxApplications < 0) { - for (String label : configuredNodeLabels) { + for (String label : queueNodeLabelsSettings.getConfiguredNodeLabels()) { int maxApplicationsByLabel = 0; if (maxGlobalPerQueueApps > 0) { // In absolute mode, should @@ -1535,7 +1298,7 @@ void deriveCapacityFromAbsoluteConfigurations(String label, // capacity = effectiveMinResource / {parent's effectiveMinResource} float result = resourceCalculator.divide(clusterResource, queueResourceQuotas.getEffectiveMinResource(label), - parent.getQueueResourceQuotas().getEffectiveMinResource(label)); + getParent().getQueueResourceQuotas().getEffectiveMinResource(label)); queueCapacities.setCapacity(label, Float.isInfinite(result) ? 0 : result); @@ -1544,7 +1307,7 @@ void deriveCapacityFromAbsoluteConfigurations(String label, // maxCapacity = effectiveMaxResource / parent's effectiveMaxResource result = resourceCalculator.divide(clusterResource, queueResourceQuotas.getEffectiveMaxResource(label), - parent.getQueueResourceQuotas().getEffectiveMaxResource(label)); + getParent().getQueueResourceQuotas().getEffectiveMaxResource(label)); queueCapacities.setMaximumCapacity(label, Float.isInfinite(result) ? 0 : result); @@ -1553,7 +1316,7 @@ void deriveCapacityFromAbsoluteConfigurations(String label, // capacity and the parent's absoluteCapacity. // absoluteCapacity = capacity * parent's absoluteCapacity queueCapacities.setAbsoluteCapacity(label, - queueCapacities.getCapacity(label) * parent.getQueueCapacities() + queueCapacities.getCapacity(label) * getParent().getQueueCapacities() .getAbsoluteCapacity(label)); // Update absolute maxCapacity (as in fraction of the @@ -1562,12 +1325,12 @@ void deriveCapacityFromAbsoluteConfigurations(String label, // absoluteMaxCapacity = maxCapacity * parent's absoluteMaxCapacity queueCapacities.setAbsoluteMaximumCapacity(label, queueCapacities.getMaximumCapacity(label) * - parent.getQueueCapacities() + getParent().getQueueCapacities() .getAbsoluteMaximumCapacity(label)); } void updateEffectiveResources(Resource clusterResource) { - for (String label : configuredNodeLabels) { + for (String label : queueNodeLabelsSettings.getConfiguredNodeLabels()) { Resource resourceByLabel = labelManager.getResourceByLabel(label, clusterResource); Resource newEffectiveMinResource; @@ -1578,7 +1341,7 @@ void updateEffectiveResources(Resource clusterResource) { CapacityConfigType.ABSOLUTE_RESOURCE)) { newEffectiveMinResource = createNormalizedMinResource( queueResourceQuotas.getConfiguredMinResource(label), - ((ParentQueue) parent).getEffectiveMinRatioPerResource()); + ((ParentQueue) getParent()).getEffectiveMinRatioPerResource()); // Max resource of a queue should be the minimum of {parent's maxResources, // this queue's maxResources}. Both parent's maxResources and this queue's @@ -1603,7 +1366,7 @@ void updateEffectiveResources(Resource clusterResource) { newEffectiveMaxResource); if (LOG.isDebugEnabled()) { - LOG.debug("Updating queue:" + queuePath + LOG.debug("Updating queue:" + getQueuePath() + " with effective minimum resource=" + newEffectiveMinResource + "and effective maximum resource=" + newEffectiveMaxResource); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/AbstractManagedParentQueue.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/AbstractManagedParentQueue.java index 9c16de097d6..ad9970c0855 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/AbstractManagedParentQueue.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/AbstractManagedParentQueue.java @@ -208,7 +208,7 @@ protected void validateQueueEntitlementChange(AbstractAutoCreatedLeafQueue if (!(newChildCap >= 0 && newChildCap < 1.0f + CSQueueUtils.EPSILON)) { throw new SchedulerDynamicEditException( "Sum of child queues should exceed 100% for auto creating parent " - + "queue : " + queueName); + + "queue : " + basicQueueProperties.getQueueName()); } } } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/BasicQueueProperties.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/BasicQueueProperties.java new file mode 100644 index 00000000000..ccfeb5202b4 --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/BasicQueueProperties.java @@ -0,0 +1,45 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * http://www.apache.org/licenses/LICENSE-2.0 + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity; + +public class BasicQueueProperties { + private final String queueName; + private final String queuePath; + private CSQueue parent; + + public BasicQueueProperties(CSQueue parent, String queueName) { + this.parent = parent; + this.queueName = queueName; + this.queuePath = ((parent == null) ? "" : (parent.getQueuePath() + ".")) + this.queueName; + } + + public String getQueuePath() { + return queuePath; + } + + public String getQueueName() { + return queueName; + } + + public void setParent(CSQueue parent) { + this.parent = parent; + } + + public CSQueue getParent() { + return parent; + } +} diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CSQueuePreemption.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CSQueuePreemptionSettings.java similarity index 96% rename from hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CSQueuePreemption.java rename to hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CSQueuePreemptionSettings.java index 0a11bcd6293..032fc5eb790 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CSQueuePreemption.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CSQueuePreemptionSettings.java @@ -18,13 +18,13 @@ import org.apache.hadoop.yarn.conf.YarnConfiguration; -public class CSQueuePreemption { +public class CSQueuePreemptionSettings { private final boolean preemptionDisabled; // Indicates if the in-queue preemption setting is ever disabled within the // hierarchy of this queue. private final boolean intraQueuePreemptionDisabledInHierarchy; - public CSQueuePreemption( + public CSQueuePreemptionSettings( CSQueue queue, CapacitySchedulerContext csContext, CapacitySchedulerConfiguration configuration) { @@ -108,6 +108,10 @@ private boolean isIntraQueueHierarchyPreemptionDisabled(CSQueue q, return configuration.getIntraQueuePreemptionDisabled(q.getQueuePath(), parentQ.getIntraQueuePreemptionDisabledInHierarchy()); } + + public boolean getIntraQueuePreemptionDisabled() { + return intraQueuePreemptionDisabledInHierarchy || preemptionDisabled; + } public boolean isIntraQueuePreemptionDisabledInHierarchy() { return intraQueuePreemptionDisabledInHierarchy; diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/LeafQueue.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/LeafQueue.java index cc282ef59a2..2cb79d60f20 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/LeafQueue.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/LeafQueue.java @@ -20,7 +20,6 @@ import java.io.IOException; import java.util.*; -import java.util.Map.Entry; import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.ConcurrentMap; @@ -216,15 +215,14 @@ protected void setupQueueConfigs(Resource clusterResource, priorityAcls = conf.getPriorityAcls(getQueuePath(), csContext.getMaxClusterLevelAppPriority()); - if (!SchedulerUtils.checkQueueLabelExpression(this.accessibleLabels, - this.defaultLabelExpression, null)) { + Set accessibleNodeLabels = this.queueNodeLabelsSettings.getAccessibleNodeLabels(); + if (!SchedulerUtils.checkQueueLabelExpression(accessibleNodeLabels, + this.queueNodeLabelsSettings.getDefaultLabelExpression(), null)) { throw new IOException( "Invalid default label expression of " + " queue=" + getQueuePath() + " doesn't have permission to access all labels " + "in default label expression. labelExpression of resource request=" - + (this.defaultLabelExpression == null ? - "" : - this.defaultLabelExpression) + ". Queue labels=" + ( + + getDefaultNodeLabelExpressionStr() + ". Queue labels=" + ( getAccessibleNodeLabels() == null ? "" : StringUtils @@ -239,8 +237,10 @@ protected void setupQueueConfigs(Resource clusterResource, // re-init this since max allocation could have changed this.minimumAllocationFactor = Resources.ratio(resourceCalculator, - Resources.subtract(maximumAllocation, minimumAllocation), - maximumAllocation); + Resources.subtract( + queueAllocationSettings.getMaximumAllocation(), + queueAllocationSettings.getMinimumAllocation()), + queueAllocationSettings.getMaximumAllocation()); StringBuilder aclsString = new StringBuilder(); for (Map.Entry e : acls.entrySet()) { @@ -248,10 +248,9 @@ protected void setupQueueConfigs(Resource clusterResource, } StringBuilder labelStrBuilder = new StringBuilder(); - if (accessibleLabels != null) { - for (String s : accessibleLabels) { - labelStrBuilder.append(s) - .append(","); + if (accessibleNodeLabels != null) { + for (String nodeLabel : accessibleNodeLabels) { + labelStrBuilder.append(nodeLabel).append(","); } } @@ -298,7 +297,8 @@ protected void setupQueueConfigs(Resource clusterResource, + "minimumAllocationFactor = " + minimumAllocationFactor + " [= (float)(maximumAllocationMemory - minimumAllocationMemory) / " + "maximumAllocationMemory ]" + "\n" + "maximumAllocation = " - + maximumAllocation + " [= configuredMaxAllocation ]" + "\n" + + queueAllocationSettings.getMaximumAllocation() + + " [= configuredMaxAllocation ]" + "\n" + "numContainers = " + numContainers + " [= currentNumContainers ]" + "\n" + "state = " + getState() + " [= configuredState ]" + "\n" + "acls = " + aclsString @@ -319,6 +319,11 @@ protected void setupQueueConfigs(Resource clusterResource, } } + private String getDefaultNodeLabelExpressionStr() { + String defaultLabelExpression = queueNodeLabelsSettings.getDefaultLabelExpression(); + return defaultLabelExpression == null ? "" : defaultLabelExpression; + } + /** * Used only by tests. */ @@ -715,6 +720,8 @@ public Resource getUserAMResourceLimitPerPartition( Resource queuePartitionResource = getEffectiveCapacity(nodePartition); + Resource minimumAllocation = queueAllocationSettings.getMinimumAllocation(); + Resource userAMLimit = Resources.multiplyAndNormalizeUp( resourceCalculator, queuePartitionResource, queueCapacities.getMaxAMResourcePercentage(nodePartition) @@ -801,7 +808,7 @@ public Resource calculateAndGetAMResourceLimitPerPartition( Resource amResouceLimit = Resources.multiplyAndNormalizeUp( resourceCalculator, queuePartitionUsableResource, amResourcePercent, - minimumAllocation); + queueAllocationSettings.getMinimumAllocation()); metrics.setAMResouceLimit(nodePartition, amResouceLimit); queueUsage.setAMLimit(nodePartition, amResouceLimit); @@ -1166,7 +1173,7 @@ public CSAssignment assignContainers(Resource clusterResource, // if our queue cannot access this node, just return if (schedulingMode == SchedulingMode.RESPECT_PARTITION_EXCLUSIVITY - && !accessibleToPartition(candidates.getPartition())) { + && !queueNodeLabelsSettings.accessibleToPartition(candidates.getPartition())) { ActivitiesLogger.QUEUE.recordQueueActivity(activitiesManager, node, getParent().getQueuePath(), getQueuePath(), ActivityState.REJECTED, ActivityDiagnosticConstant.QUEUE_NOT_ABLE_TO_ACCESS_PARTITION); @@ -1459,8 +1466,8 @@ public void apply(Resource cluster, writeLock.unlock(); } - if (parent != null && applyToParentQueue) { - parent.apply(cluster, request); + if (getParent() != null && applyToParentQueue) { + getParent().apply(cluster, request); } } @@ -1517,7 +1524,8 @@ private Resource getHeadroom(User user, queueUsage.getUsed(partition))); // Normalize it before return headroom = - Resources.roundDown(resourceCalculator, headroom, minimumAllocation); + Resources.roundDown(resourceCalculator, headroom, + queueAllocationSettings.getMinimumAllocation()); //headroom = min (unused resourcelimit of a label, calculated headroom ) Resource clusterPartitionResource = @@ -1923,7 +1931,7 @@ private void updateCurrentResourceLimits( this.cachedResourceLimitsForHeadroom = new ResourceLimits(currentResourceLimits.getLimit()); Resource queueMaxResource = getEffectiveMaxCapacityDown( - RMNodeLabelsManager.NO_LABEL, minimumAllocation); + RMNodeLabelsManager.NO_LABEL, queueAllocationSettings.getMinimumAllocation()); this.cachedResourceLimitsForHeadroom.setLimit(Resources.min( resourceCalculator, clusterResource, queueMaxResource, currentResourceLimits.getLimit())); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/ManagedParentQueue.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/ManagedParentQueue.java index e415ac12795..ecf24032908 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/ManagedParentQueue.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/ManagedParentQueue.java @@ -122,7 +122,7 @@ public void reinitialize(CSQueue newlyParsedQueue, Resource clusterResource) LOG.info( "Reinitialized Managed Parent Queue: [{}] with capacity [{}]" + " with max capacity [{}]", - queueName, super.getCapacity(), super.getMaximumCapacity()); + basicQueueProperties.getQueueName(), super.getCapacity(), super.getMaximumCapacity()); } catch (YarnException ye) { LOG.error("Exception while computing policy changes for leaf queue : " + getQueuePath(), ye); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/ParentQueue.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/ParentQueue.java index 3a25a17d975..0a92574d545 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/ParentQueue.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/ParentQueue.java @@ -169,21 +169,20 @@ protected void setupQueueConfigs(Resource clusterResource, } StringBuilder labelStrBuilder = new StringBuilder(); - if (accessibleLabels != null) { - for (String s : accessibleLabels) { - labelStrBuilder.append(s) - .append(","); + if (queueNodeLabelsSettings.getAccessibleNodeLabels() != null) { + for (String nodeLabel : queueNodeLabelsSettings.getAccessibleNodeLabels()) { + labelStrBuilder.append(nodeLabel).append(","); } } // Initialize queue ordering policy queueOrderingPolicy = csConf.getQueueOrderingPolicy( - getQueuePath(), parent == null ? + getQueuePath(), getParent() == null ? null : - ((ParentQueue) parent).getQueueOrderingPolicyConfigName()); + ((ParentQueue) getParent()).getQueueOrderingPolicyConfigName()); queueOrderingPolicy.setQueues(childQueues); - LOG.info(queueName + ", " + getCapacityOrWeightString() + LOG.info(getQueueName() + ", " + getCapacityOrWeightString() + ", absoluteCapacity=" + this.queueCapacities.getAbsoluteCapacity() + ", maxCapacity=" + this.queueCapacities.getMaximumCapacity() + ", absoluteMaxCapacity=" + this.queueCapacities @@ -334,7 +333,7 @@ void setChildQueues(Collection childQueues) throws IOException { throw new IOException( "Parent Queues" + " capacity: " + parentMinResource + " is less than" + " to its children:" + minRes - + " for queue:" + queueName); + + " for queue:" + getQueueName()); } } } @@ -356,7 +355,7 @@ void setChildQueues(Collection childQueues) throws IOException { // It is wrong when percent sum != {0, 1} throw new IOException( "Illegal" + " capacity sum of " + childrenPctSum - + " for children of queue " + queueName + " for label=" + + " for children of queue " + getQueueName() + " for label=" + nodeLabel + ". It should be either 0 or 1.0"); } else{ // We also allow children's percent sum = 0 under the following @@ -369,7 +368,7 @@ void setChildQueues(Collection childQueues) throws IOException { > PRECISION) && (!allowZeroCapacitySum)) { throw new IOException( "Illegal" + " capacity sum of " + childrenPctSum - + " for children of queue " + queueName + + " for children of queue " + getQueueName() + " for label=" + nodeLabel + ". It is set to 0, but parent percent != 0, and " + "doesn't allow children capacity to set to 0"); @@ -384,8 +383,8 @@ void setChildQueues(Collection childQueues) throws IOException { && !allowZeroCapacitySum) { throw new IOException( "Illegal" + " capacity sum of " + childrenPctSum - + " for children of queue " + queueName + " for label=" - + nodeLabel + ". queue=" + queueName + + " for children of queue " + getQueueName() + " for label=" + + nodeLabel + ". queue=" + getQueueName() + " has zero capacity, but child" + "queues have positive capacities"); } @@ -471,7 +470,7 @@ private QueueUserACLInfo getUserAclInfo( } public String toString() { - return queueName + ": " + + return getQueueName() + ": " + "numChildQueue= " + childQueues.size() + ", " + getCapacityOrWeightString() + ", " + "absoluteCapacity=" + queueCapacities.getAbsoluteCapacity() + ", " + @@ -753,12 +752,12 @@ public void submitApplication(ApplicationId applicationId, String user, } // Inform the parent queue - if (parent != null) { + if (getParent() != null) { try { - parent.submitApplication(applicationId, user, queue); + getParent().submitApplication(applicationId, user, queue); } catch (AccessControlException ace) { - LOG.info("Failed to submit application to parent-queue: " + - parent.getQueuePath(), ace); + LOG.info("Failed to submit application to parent-queue: " + + getParent().getQueuePath(), ace); removeApplication(applicationId, user); throw ace; } @@ -769,9 +768,9 @@ public void validateSubmitApplication(ApplicationId applicationId, String userName, String queue) throws AccessControlException { writeLock.lock(); try { - if (queue.equals(queueName)) { + if (queue.equals(getQueueName())) { throw new AccessControlException( - "Cannot submit application " + "to non-leaf queue: " + queueName); + "Cannot submit application " + "to non-leaf queue: " + getQueueName()); } if (getState() != QueueState.RUNNING) { @@ -826,8 +825,8 @@ public void finishApplication(ApplicationId application, String user) { appFinished(); // Inform the parent queue - if (parent != null) { - parent.finishApplication(application, user); + if (getParent() != null) { + getParent().finishApplication(application, user); } } @@ -857,7 +856,7 @@ public CSAssignment assignContainers(Resource clusterResource, // if our queue cannot access this node, just return if (schedulingMode == SchedulingMode.RESPECT_PARTITION_EXCLUSIVITY - && !accessibleToPartition(candidates.getPartition())) { + && !queueNodeLabelsSettings.accessibleToPartition(candidates.getPartition())) { if (LOG.isDebugEnabled()) { long now = System.currentTimeMillis(); // Do logging every 1 sec to avoid excessive logging. @@ -1038,7 +1037,7 @@ private boolean canAssign(Resource clusterResource, FiCaSchedulerNode node) { // 1) Node doesn't have reserved container // 2) Node's available-resource + killable-resource should > 0 boolean accept = node.getReservedContainer() == null && - Resources.fitsIn(resourceCalculator, minimumAllocation, + Resources.fitsIn(resourceCalculator, queueAllocationSettings.getMinimumAllocation(), Resources.add(node.getUnallocatedResource(), node.getTotalKillableResources())); if (!accept) { ActivitiesLogger.QUEUE.recordQueueActivity(activitiesManager, node, @@ -1085,7 +1084,8 @@ private ResourceLimits getResourceLimitsOfChild(CSQueue child, // Normalize before return childLimit = - Resources.roundDown(resourceCalculator, childLimit, minimumAllocation); + Resources.roundDown(resourceCalculator, childLimit, + queueAllocationSettings.getMinimumAllocation()); return new ResourceLimits(childLimit); } @@ -1198,9 +1198,9 @@ public void completedContainer(Resource clusterResource, rmContainer.getContainer().getResource()); // Inform the parent - if (parent != null) { + if (getParent() != null) { // complete my parent - parent.completedContainer(clusterResource, application, + getParent().completedContainer(clusterResource, application, node, rmContainer, null, event, this, sortQueues); } } @@ -1270,7 +1270,7 @@ public void updateClusterResource(Resource clusterResource, } // Update effective capacity in all parent queue. - for (String label : configuredNodeLabels) { + for (String label : queueNodeLabelsSettings.getConfiguredNodeLabels()) { calculateEffectiveResourcesAndCapacity(label, clusterResource); } @@ -1407,8 +1407,8 @@ public void recoverContainer(Resource clusterResource, writeLock.unlock(); } - if (parent != null) { - parent.recoverContainer(clusterResource, attempt, rmContainer); + if (getParent() != null) { + getParent().recoverContainer(clusterResource, attempt, rmContainer); } } @@ -1445,8 +1445,8 @@ public void attachContainer(Resource clusterResource, + getAbsoluteUsedCapacity() + " used=" + queueUsage.getUsed() + " cluster=" + clusterResource); // Inform the parent - if (parent != null) { - parent.attachContainer(clusterResource, application, rmContainer); + if (getParent() != null) { + getParent().attachContainer(clusterResource, application, rmContainer); } } } @@ -1465,8 +1465,8 @@ public void detachContainer(Resource clusterResource, + getAbsoluteUsedCapacity() + " used=" + queueUsage.getUsed() + " cluster=" + clusterResource); // Inform the parent - if (parent != null) { - parent.detachContainer(clusterResource, application, rmContainer); + if (getParent() != null) { + getParent().detachContainer(clusterResource, application, rmContainer); } } } @@ -1592,8 +1592,8 @@ public void apply(Resource cluster, } } - if (parent != null) { - parent.apply(cluster, request); + if (getParent() != null) { + getParent().apply(cluster, request); } } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/QueueAllocationSettings.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/QueueAllocationSettings.java new file mode 100644 index 00000000000..9eb1eefdb6e --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/QueueAllocationSettings.java @@ -0,0 +1,92 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * http://www.apache.org/licenses/LICENSE-2.0 + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity; + +import org.apache.hadoop.yarn.api.records.Resource; +import org.apache.hadoop.yarn.api.records.ResourceInformation; +import org.apache.hadoop.yarn.util.resource.ResourceUtils; +import org.apache.hadoop.yarn.util.resource.Resources; + +import static org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacitySchedulerConfiguration.UNDEFINED; + +public class QueueAllocationSettings { + private final Resource minimumAllocation; + private volatile Resource maximumAllocation; + + public QueueAllocationSettings(CapacitySchedulerContext csContext) { + this.minimumAllocation = csContext.getMinimumResourceCapability(); + } + + void setupMaximumAllocation(CapacitySchedulerConfiguration csConf, String queuePath, + CSQueue parent, CapacitySchedulerContext csContext) { + /* YARN-10869: When using AutoCreatedLeafQueues, the passed configuration + * object is a cloned one containing only the template configs + * (see ManagedParentQueue#getLeafQueueConfigs). To ensure that the actual + * cluster maximum allocation is fetched the original config object should + * be used. + */ + Resource clusterMax = ResourceUtils + .fetchMaximumAllocationFromConfig(csContext.getConfiguration()); + Resource queueMax = csConf.getQueueMaximumAllocation(queuePath); + + maximumAllocation = Resources.clone( + parent == null ? clusterMax : parent.getMaximumAllocation()); + + String errMsg = + "Queue maximum allocation cannot be larger than the cluster setting" + + " for queue " + queuePath + + " max allocation per queue: %s" + + " cluster setting: " + clusterMax; + + if (queueMax == Resources.none()) { + // Handle backward compatibility + long queueMemory = csConf.getQueueMaximumAllocationMb(queuePath); + int queueVcores = csConf.getQueueMaximumAllocationVcores(queuePath); + if (queueMemory != UNDEFINED) { + maximumAllocation.setMemorySize(queueMemory); + } + + if (queueVcores != UNDEFINED) { + maximumAllocation.setVirtualCores(queueVcores); + } + + if ((queueMemory != UNDEFINED && queueMemory > clusterMax.getMemorySize() + || (queueVcores != UNDEFINED + && queueVcores > clusterMax.getVirtualCores()))) { + throw new IllegalArgumentException( + String.format(errMsg, maximumAllocation)); + } + } else { + // Queue level maximum-allocation can't be larger than cluster setting + for (ResourceInformation ri : queueMax.getResources()) { + if (ri.compareTo(clusterMax.getResourceInformation(ri.getName())) > 0) { + throw new IllegalArgumentException(String.format(errMsg, queueMax)); + } + + maximumAllocation.setResourceInformation(ri.getName(), ri); + } + } + } + + public Resource getMinimumAllocation() { + return minimumAllocation; + } + + public Resource getMaximumAllocation() { + return maximumAllocation; + } +} diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/QueueAppLifetimeAndLimitSettings.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/QueueAppLifetimeAndLimitSettings.java new file mode 100644 index 00000000000..d611deed726 --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/QueueAppLifetimeAndLimitSettings.java @@ -0,0 +1,127 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * http://www.apache.org/licenses/LICENSE-2.0 + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity; + +import org.apache.hadoop.yarn.exceptions.YarnRuntimeException; + +public class QueueAppLifetimeAndLimitSettings { + // -1 indicates lifetime is disabled + private volatile long maxApplicationLifetime = -1; + private volatile long defaultApplicationLifetime = -1; + + // Indicates if this queue's default lifetime was set by a config property, + // either at this level or anywhere in the queue's hierarchy. + private volatile boolean defaultAppLifetimeWasSpecifiedInConfig = false; + + private int maxParallelApps; + + public QueueAppLifetimeAndLimitSettings(CapacitySchedulerConfiguration configuration, + AbstractCSQueue q, String queuePath) { + // Store max parallel apps property + this.maxParallelApps = configuration.getMaxParallelAppsForQueue(queuePath); + this.maxApplicationLifetime = getInheritedMaxAppLifetime(q, configuration); + this.defaultApplicationLifetime = getInheritedDefaultAppLifetime(q, queuePath, configuration, + maxApplicationLifetime); + } + + private long getInheritedMaxAppLifetime(CSQueue q, CapacitySchedulerConfiguration conf) { + CSQueue parentQ = q.getParent(); + long maxAppLifetime = conf.getMaximumLifetimePerQueue(q.getQueuePath()); + + // If q is the root queue, then get max app lifetime from conf. + if (parentQ == null) { + return maxAppLifetime; + } + + // If this is not the root queue, get this queue's max app lifetime + // from the conf. The parent's max app lifetime will be used if it's + // not set for this queue. + // A value of 0 will override the parent's value and means no max lifetime. + // A negative value means that the parent's max should be used. + long parentsMaxAppLifetime = parentQ.getMaximumApplicationLifetime(); + return (maxAppLifetime >= 0) ? maxAppLifetime : parentsMaxAppLifetime; + } + + private long getInheritedDefaultAppLifetime(CSQueue q, + String queuePath, CapacitySchedulerConfiguration conf, long myMaxAppLifetime) { + CSQueue parentQ = q.getParent(); + long defaultAppLifetime = conf.getDefaultLifetimePerQueue(queuePath); + defaultAppLifetimeWasSpecifiedInConfig = + (defaultAppLifetime >= 0 + || (parentQ != null && + parentQ.getDefaultAppLifetimeWasSpecifiedInConfig())); + + // If q is the root queue, then get default app lifetime from conf. + if (parentQ == null) { + return defaultAppLifetime; + } + + // If this is not the root queue, get the parent's default app lifetime. The + // parent's default app lifetime will be used if not set for this queue. + long parentsDefaultAppLifetime = parentQ.getDefaultApplicationLifetime(); + + // Negative value indicates default lifetime was not set at this level. + // If default lifetime was not set at this level, calculate it based on + // parent's default lifetime or current queue's max lifetime. + if (defaultAppLifetime < 0) { + // If default lifetime was not set at this level but was set somewhere in + // the parent's hierarchy, set default lifetime to parent queue's default + // only if parent queue's lifetime is less than current queue's max + // lifetime. Otherwise, use current queue's max lifetime value for its + // default lifetime. + if (defaultAppLifetimeWasSpecifiedInConfig) { + defaultAppLifetime = + Math.min(parentsDefaultAppLifetime, myMaxAppLifetime); + } else { + // Default app lifetime value was not set anywhere in this queue's + // hierarchy. Use current queue's max lifetime as its default. + defaultAppLifetime = myMaxAppLifetime; + } + } // else if >= 0, default lifetime was set at this level. Just use it. + + if (myMaxAppLifetime > 0 && defaultAppLifetime > myMaxAppLifetime) { + throw new YarnRuntimeException( + "Default lifetime " + defaultAppLifetime + + " can't exceed maximum lifetime " + myMaxAppLifetime); + } + + if (defaultAppLifetime <= 0) { + defaultAppLifetime = myMaxAppLifetime; + } + return defaultAppLifetime; + } + + public int getMaxParallelApps() { + return maxParallelApps; + } + + public void setMaxParallelApps(int maxParallelApps) { + this.maxParallelApps = maxParallelApps; + } + + public long getMaxApplicationLifetime() { + return maxApplicationLifetime; + } + + public long getDefaultApplicationLifetime() { + return defaultApplicationLifetime; + } + + public boolean isDefaultAppLifetimeWasSpecifiedInConfig() { + return defaultAppLifetimeWasSpecifiedInConfig; + } +} diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/QueueNodeLabelsSettings.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/QueueNodeLabelsSettings.java new file mode 100644 index 00000000000..269f29bdc3b --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/QueueNodeLabelsSettings.java @@ -0,0 +1,142 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * http://www.apache.org/licenses/LICENSE-2.0 + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity; + +import org.apache.commons.lang3.StringUtils; +import org.apache.hadoop.util.Sets; +import org.apache.hadoop.yarn.server.resourcemanager.nodelabels.RMNodeLabelsManager; +import static org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacitySchedulerConfiguration.ROOT; +import java.io.IOException; +import java.util.Set; + +public class QueueNodeLabelsSettings { + private final CSQueue parent; + private final String queuePath; + private final CapacitySchedulerContext csContext; + private Set accessibleLabels; + private Set configuredNodeLabels; + private String defaultLabelExpression; + + public QueueNodeLabelsSettings(CapacitySchedulerConfiguration configuration, + CSQueue parent, + String queuePath, + CapacitySchedulerContext csContext) throws IOException { + this.parent = parent; + this.queuePath = queuePath; + this.csContext = csContext; + initializeNodeLabels(configuration); + } + + private void initializeNodeLabels(CapacitySchedulerConfiguration configuration) + throws IOException { + initializeAccessibleLabels(configuration); + initializeDefaultLabelExpression(configuration); + initializeConfiguredNodeLabels(); + validateNodeLabels(); + } + + private void initializeAccessibleLabels(CapacitySchedulerConfiguration configuration) { + this.accessibleLabels = configuration.getAccessibleNodeLabels(queuePath); + // Inherit labels from parent if not set + if (this.accessibleLabels == null && parent != null) { + this.accessibleLabels = parent.getAccessibleNodeLabels(); + } + } + + private void initializeDefaultLabelExpression(CapacitySchedulerConfiguration configuration) { + this.defaultLabelExpression = configuration.getDefaultNodeLabelExpression(queuePath); + // If the accessible labels is not null and the queue has a parent with a + // similar set of labels copy the defaultNodeLabelExpression from the parent + if (this.accessibleLabels != null && parent != null + && this.defaultLabelExpression == null && + this.accessibleLabels.containsAll(parent.getAccessibleNodeLabels())) { + this.defaultLabelExpression = parent.getDefaultNodeLabelExpression(); + } + } + + private void initializeConfiguredNodeLabels() { + if (csContext.getCapacitySchedulerQueueManager() != null + && csContext.getCapacitySchedulerQueueManager().getConfiguredNodeLabels() != null) { + if (queuePath.equals(ROOT)) { + this.configuredNodeLabels = csContext.getCapacitySchedulerQueueManager() + .getConfiguredNodeLabels().getAllConfiguredLabels(); + } else { + this.configuredNodeLabels = csContext.getCapacitySchedulerQueueManager() + .getConfiguredNodeLabels().getLabelsByQueue(queuePath); + } + } else { + // Fallback to suboptimal but correct logic + this.configuredNodeLabels = csContext.getConfiguration().getConfiguredNodeLabels(queuePath); + } + } + + private void validateNodeLabels() throws IOException { + // Check if labels of this queue is a subset of parent queue, only do this + // when the queue in question is not root + if (isNotRoot()) { + if (parent.getAccessibleNodeLabels() != null && !parent + .getAccessibleNodeLabels().contains(RMNodeLabelsManager.ANY)) { + // If parent isn't "*", child shouldn't be "*" too + if (this.getAccessibleNodeLabels().contains(RMNodeLabelsManager.ANY)) { + throw new IOException("Parent's accessible queue is not ANY(*), " + + "but child's accessible queue is " + RMNodeLabelsManager.ANY); + } else { + Set diff = Sets.difference(this.getAccessibleNodeLabels(), + parent.getAccessibleNodeLabels()); + if (!diff.isEmpty()) { + throw new IOException(String.format( + "Some labels of child queue is not a subset of parent queue, these labels=[%s]", + StringUtils.join(diff, ","))); + } + } + } + } + } + + private boolean isNotRoot() { + return parent != null && parent.getParent() != null; + } + + public boolean accessibleToPartition(String nodePartition) { + // if queue's label is *, it can access any node + if (accessibleLabels != null && accessibleLabels.contains(RMNodeLabelsManager.ANY)) { + return true; + } + // any queue can access to a node without label + if (nodePartition == null || nodePartition.equals(RMNodeLabelsManager.NO_LABEL)) { + return true; + } + // a queue can access to a node only if it contains any label of the node + if (accessibleLabels != null && accessibleLabels.contains(nodePartition)) { + return true; + } + // The partition cannot be accessed + return false; + } + + public Set getAccessibleNodeLabels() { + return accessibleLabels; + } + + public Set getConfiguredNodeLabels() { + return configuredNodeLabels; + } + + public String getDefaultLabelExpression() { + return defaultLabelExpression; + } +} diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestLeafQueue.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestLeafQueue.java index 351fdbe3a53..0a0b682a2b1 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestLeafQueue.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestLeafQueue.java @@ -5245,7 +5245,7 @@ public void testRootHasAllNodeLabelsOfItsDescendants() throws IOException { ParentQueue rootQueue = (ParentQueue) cs.getRootQueue(); Assert.assertEquals(Sets.newHashSet("", "test", "test2"), - rootQueue.configuredNodeLabels); + rootQueue.queueNodeLabelsSettings.getConfiguredNodeLabels()); } @After