diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/placement/ApplicationPlacementContext.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/placement/ApplicationPlacementContext.java index f2f92b81fbc..a79c3db4168 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/placement/ApplicationPlacementContext.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/placement/ApplicationPlacementContext.java @@ -29,6 +29,11 @@ private String parentQueue; + // TODO, make this set by constructor + private boolean createLeafQueue = false; + + private boolean createParentQueue = false; + public ApplicationPlacementContext(String queue) { this(queue,null); } @@ -42,6 +47,10 @@ public String getQueue() { return queue; } + public void setQueue(String q) { + queue = q; + } + public String getParentQueue() { return parentQueue; } @@ -49,4 +58,43 @@ public String getParentQueue() { public boolean hasParentQueue() { return parentQueue != null; } + + public String getFullQueuePath() { + if (parentQueue != null) { + return parentQueue + "." + queue; + } else { + return queue; + } + } + + public boolean isCreateLeafQueue() { + return createLeafQueue; + } + + public boolean isCreateParentQueue() { + return createParentQueue; + } + + public void setCreateLeafQueue(boolean createLeafQueue) { + this.createLeafQueue = createLeafQueue; + } + + public void setCreateParentQueue(boolean createParentQueue) { + this.createParentQueue = createParentQueue; + } + + @Override + public String toString() { + return "ApplicationPlacementContext{" + "queue='" + queue + '\'' + + ", parentQueue='" + parentQueue + '\'' + ", createLeafQueue=" + + createLeafQueue + ", createParentQueue=" + createParentQueue + '}'; + } + + public ApplicationPlacementContext clone() { + ApplicationPlacementContext apc = new ApplicationPlacementContext(this.queue, + this.parentQueue); + apc.setCreateLeafQueue(this.createLeafQueue); + apc.setCreateParentQueue(this.createParentQueue); + return apc; + } } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/AbstractAutoCreatedLeafQueue.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/AbstractAutoCreatedLeafQueue.java index 2b222419606..8d7733453f8 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/AbstractAutoCreatedLeafQueue.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/AbstractAutoCreatedLeafQueue.java @@ -128,9 +128,4 @@ public void setEntitlement(String nodeLabel, QueueEntitlement entitlement) writeLock.unlock(); } } - - protected void setupConfigurableCapacities(QueueCapacities queueCapacities) { - CSQueueUtils.updateAndCheckCapacitiesByLabel(getQueuePath(), - queueCapacities, parent == null ? null : parent.getQueueCapacities()); - } } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/AbstractCSQueue.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/AbstractCSQueue.java index 8d22a36d99d..28d6e8b06e7 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/AbstractCSQueue.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/AbstractCSQueue.java @@ -100,7 +100,7 @@ String defaultLabelExpression; private String multiNodeSortingPolicyName = null; - Map acls = + Map acls = new HashMap(); volatile boolean reservationsContinueLooking; private volatile boolean preemptionDisabled; @@ -112,7 +112,7 @@ volatile ResourceUsage queueUsage; private final boolean fullPathQueueNamingPolicy = false; - + // Track capacities like used-capcity/abs-used-capacity/capacity/abs-capacity, // etc. QueueCapacities queueCapacities; @@ -129,12 +129,15 @@ private volatile boolean defaultAppLifetimeWasSpecifiedInConfig = false; protected enum CapacityConfigType { + // FIXME, from what I can see, Percentage mode can almost apply to weighted + // and percentage mode at the same time, there's only small area need to be + // changed, we need to rename "PERCENTAGE" to "PERCENTAGE" and "WEIGHT" NONE, PERCENTAGE, ABSOLUTE_RESOURCE }; protected CapacityConfigType capacityConfigType = CapacityConfigType.NONE; - private final RecordFactory recordFactory = + private final RecordFactory recordFactory = RecordFactoryProvider.getRecordFactory(null); protected CapacitySchedulerContext csContext; protected YarnAuthorizationProvider authorizer = null; @@ -148,6 +151,13 @@ private Map userWeights = new HashMap(); private int maxParallelApps; + // is it a dynamic queue? + private boolean dynamicQueue = false; + + // When this queue has application submit to? This property only applies to dynamic queue, + // and will be used to check when the queue need to be removed. + private long lastSubmittedTimestamp; + public AbstractCSQueue(CapacitySchedulerContext cs, String queueName, CSQueue parent, CSQueue old) throws IOException { this(cs, cs.getConfiguration(), queueName, parent, old); @@ -169,7 +179,7 @@ public AbstractCSQueue(CapacitySchedulerContext cs, this.metrics = old != null ? (CSQueueMetrics) old.getMetrics() : CSQueueMetrics.forQueue(getQueuePath(), parent, - cs.getConfiguration().getEnableUserMetrics(), cs.getConf()); + cs.getConfiguration().getEnableUserMetrics(), configuration); this.csContext = cs; this.minimumAllocation = csContext.getMinimumResourceCapability(); @@ -189,17 +199,15 @@ public AbstractCSQueue(CapacitySchedulerContext cs, writeLock = lock.writeLock(); } + @VisibleForTesting protected void setupConfigurableCapacities() { setupConfigurableCapacities(csContext.getConfiguration()); } protected void setupConfigurableCapacities( CapacitySchedulerConfiguration configuration) { - CSQueueUtils.loadUpdateAndCheckCapacities( - getQueuePath(), - configuration, - queueCapacities, - parent == null ? null : parent.getQueueCapacities()); + CSQueueUtils.loadCapacitiesByLabelsFromConf(getQueuePath(), queueCapacities, + configuration); } @Override @@ -250,12 +258,12 @@ public int getNumContainers() { public QueueState getState() { return state; } - + @Override public CSQueueMetrics getMetrics() { return metrics; } - + @Override public String getQueueShortName() { return queueName; @@ -283,7 +291,7 @@ public CSQueue getParent() { public void setParent(CSQueue newParentQueue) { this.parent = newParentQueue; } - + public Set getAccessibleNodeLabels() { return accessibleLabels; } @@ -344,11 +352,6 @@ void setMaxCapacity(String nodeLabel, float maximumCapacity) { public String getDefaultNodeLabelExpression() { return defaultLabelExpression; } - - void setupQueueConfigs(Resource clusterResource) - throws IOException { - setupQueueConfigs(clusterResource, csContext.getConfiguration()); - } protected void setupQueueConfigs(Resource clusterResource, CapacitySchedulerConfiguration configuration) throws @@ -381,6 +384,7 @@ protected void setupQueueConfigs(Resource clusterResource, // After we setup labels, we can setup capacities setupConfigurableCapacities(configuration); + updateAbsoluteCapacities(); // Also fetch minimum/maximum resource constraint for this queue if // configured. @@ -404,7 +408,7 @@ protected void setupQueueConfigs(Resource clusterResource, QueueState parentState = (parent == null) ? null : parent.getState(); initializeQueueState(previous, configuredState, parentState); - authorizer = YarnAuthorizationProvider.getInstance(csContext.getConf()); + authorizer = YarnAuthorizationProvider.getInstance(configuration); this.acls = configuration.getAcls(getQueuePath()); @@ -436,7 +440,7 @@ protected void setupQueueConfigs(Resource clusterResource, } this.reservationsContinueLooking = - csContext.getConfiguration().getReservationContinueLook(); + configuration.getReservationContinueLook(); this.preemptionDisabled = isQueueHierarchyPreemptionDisabled(this, configuration); @@ -472,14 +476,14 @@ protected void setupQueueConfigs(Resource clusterResource, private void setupMaximumAllocation(CapacitySchedulerConfiguration csConf) { String myQueuePath = getQueuePath(); Resource clusterMax = ResourceUtils - .fetchMaximumAllocationFromConfig(csConf); + .fetchMaximumAllocationFromConfig(csConf); Resource queueMax = csConf.getQueueMaximumAllocation(myQueuePath); maximumAllocation = Resources.clone( - parent == null ? clusterMax : parent.getMaximumAllocation()); + parent == null ? clusterMax : parent.getMaximumAllocation()); String errMsg = - "Queue maximum allocation cannot be larger than the cluster setting" + "Queue maximum allocation cannot be larger than the cluster setting" + " for queue " + myQueuePath + " max allocation per queue: %s" + " cluster setting: " + clusterMax; @@ -498,9 +502,9 @@ private void setupMaximumAllocation(CapacitySchedulerConfiguration csConf) { if ((queueMemory != UNDEFINED && queueMemory > clusterMax.getMemorySize() || (queueVcores != UNDEFINED - && queueVcores > clusterMax.getVirtualCores()))) { + && queueVcores > clusterMax.getVirtualCores()))) { throw new IllegalArgumentException( - String.format(errMsg, maximumAllocation)); + String.format(errMsg, maximumAllocation)); } } else { // Queue level maximum-allocation can't be larger than cluster setting @@ -562,7 +566,7 @@ protected void updateConfigurableResourceRequirement(String queuePath, CapacityConfigType localType = checkConfigTypeIsAbsoluteResource( queuePath, label) ? CapacityConfigType.ABSOLUTE_RESOURCE - : CapacityConfigType.PERCENTAGE; + : CapacityConfigType.PERCENTAGE; if (this.capacityConfigType.equals(CapacityConfigType.NONE)) { this.capacityConfigType = localType; @@ -605,7 +609,7 @@ protected void updateConfigurableResourceRequirement(String queuePath, } LOG.debug("Updating absolute resource configuration for queue:{} as" - + " minResource={} and maxResource={}", getQueuePath(), minResource, + + " minResource={} and maxResource={}", getQueuePath(), minResource, maxResource); queueResourceQuotas.setConfiguredMinResource(label, minResource); @@ -680,8 +684,8 @@ private void initializeQueueState(QueueState previousState, && parentState != QueueState.RUNNING) { throw new IllegalArgumentException( "The parent queue:" + parent.getQueuePath() - + " cannot be STOPPED as the child queue:" + queuePath - + " is in RUNNING state."); + + " cannot be STOPPED as the child queue:" + queuePath + + " is in RUNNING state."); } else { updateQueueState(configuredState); } @@ -752,7 +756,7 @@ public QueueStatistics getQueueStatistics() { stats.setReservedContainers(getMetrics().getReservedContainers()); return stats; } - + public Map getQueueConfigurations() { Map queueConfigurations = new HashMap<>(); Set nodeLabels = getNodeLabelsForQueue(); @@ -788,12 +792,12 @@ public QueueStatistics getQueueStatistics() { public Resource getMaximumAllocation() { return maximumAllocation; } - + @Private public Resource getMinimumAllocation() { return minimumAllocation; } - + void allocateResource(Resource clusterResource, Resource resource, String nodePartition) { writeLock.lock(); @@ -808,7 +812,7 @@ void allocateResource(Resource clusterResource, writeLock.unlock(); } } - + protected void releaseResource(Resource clusterResource, Resource resource, String nodePartition) { writeLock.lock(); @@ -823,12 +827,12 @@ protected void releaseResource(Resource clusterResource, writeLock.unlock(); } } - + @Private public boolean getReservationContinueLooking() { return reservationsContinueLooking; } - + @Private public Map getACLs() { readLock.lock(); @@ -853,12 +857,12 @@ public boolean getIntraQueuePreemptionDisabled() { public boolean getIntraQueuePreemptionDisabledInHierarchy() { return intraQueuePreemptionDisabledInHierarchy; } - + @Private public QueueCapacities getQueueCapacities() { return queueCapacities; } - + @Private public ResourceUsage getQueueResourceUsage() { return queueUsage; @@ -889,7 +893,7 @@ private boolean isQueueHierarchyPreemptionDisabled(CSQueue q, boolean systemWidePreemption = csContext.getConfiguration() .getBoolean(YarnConfiguration.RM_SCHEDULER_ENABLE_MONITORS, - YarnConfiguration.DEFAULT_RM_SCHEDULER_ENABLE_MONITORS); + YarnConfiguration.DEFAULT_RM_SCHEDULER_ENABLE_MONITORS); CSQueue parentQ = q.getParent(); // If the system-wide preemption switch is turned off, all of the queues in @@ -908,7 +912,7 @@ private boolean isQueueHierarchyPreemptionDisabled(CSQueue q, // inherited from the parent's hierarchy unless explicitly overridden at // this level. return configuration.getPreemptionDisabled(q.getQueuePath(), - parentQ.getPreemptionDisabled()); + parentQ.getPreemptionDisabled()); } private long getInheritedMaxAppLifetime(CSQueue q, @@ -936,7 +940,7 @@ private long getInheritedDefaultAppLifetime(CSQueue q, long defaultAppLifetime = conf.getDefaultLifetimePerQueue(getQueuePath()); defaultAppLifetimeWasSpecifiedInConfig = (defaultAppLifetime >= 0 - || (parentQ != null && + || (parentQ != null && parentQ.getDefaultAppLifetimeWasSpecifiedInConfig())); // If q is the root queue, then get default app lifetime from conf. @@ -990,7 +994,7 @@ private boolean isIntraQueueHierarchyPreemptionDisabled(CSQueue q, csContext.getConfiguration().getBoolean( CapacitySchedulerConfiguration.INTRAQUEUE_PREEMPTION_ENABLED, CapacitySchedulerConfiguration - .DEFAULT_INTRAQUEUE_PREEMPTION_ENABLED); + .DEFAULT_INTRAQUEUE_PREEMPTION_ENABLED); // Intra-queue preemption is disabled for this queue if the system-wide // intra-queue preemption flag is false if (!systemWideIntraQueuePreemption) return true; @@ -1030,7 +1034,7 @@ private Resource getCurrentLimitResource(String nodePartition, // all queues on this label equals to total resource with the label. return labelManager.getResourceByLabel(nodePartition, clusterResource); } - + return Resources.none(); } @@ -1083,7 +1087,7 @@ boolean canAssignToThisQueue(Resource clusterResource, // has reserved containers. if (this.reservationsContinueLooking && Resources.greaterThan(resourceCalculator, clusterResource, - resourceCouldBeUnreserved, Resources.none())) { + resourceCouldBeUnreserved, Resources.none())) { // resource-without-reserved = used - reserved Resource newTotalWithoutReservedResource = Resources.subtract( usedExceptKillable, resourceCouldBeUnreserved); @@ -1171,7 +1175,7 @@ public void incPendingResource(String nodeLabel, Resource resourceToInc) { parent.incPendingResource(nodeLabel, resourceToInc); } } - + @Override public void decPendingResource(String nodeLabel, Resource resourceToDec) { if (nodeLabel == null) { @@ -1183,7 +1187,7 @@ public void decPendingResource(String nodeLabel, Resource resourceToDec) { parent.decPendingResource(nodeLabel, resourceToDec); } } - + @Override public void incUsedResource(String nodeLabel, Resource resourceToInc, SchedulerApplicationAttempt application) { @@ -1218,14 +1222,14 @@ public void decUsedResource(String nodeLabel, Resource resourceToDec, /** * Return if the queue has pending resource on given nodePartition and - * schedulingMode. + * schedulingMode. */ - boolean hasPendingResourceRequest(String nodePartition, + boolean hasPendingResourceRequest(String nodePartition, Resource cluster, SchedulingMode schedulingMode) { return SchedulerUtils.hasPendingResourceRequest(resourceCalculator, queueUsage, nodePartition, cluster, schedulingMode); } - + public boolean accessibleToPartition(String nodePartition) { // if queue's label is *, it can access any node if (accessibleLabels != null @@ -1447,4 +1451,199 @@ public int getMaxParallelApps() { } abstract int getNumRunnableApps(); + + protected void updateAbsoluteCapacities() { + QueueCapacities parentQueueCapacities = null; + if (parent != null) { + parentQueueCapacities = parent.getQueueCapacities(); + } + + CSQueueUtils.updateAbsoluteCapacitiesByNodeLabels(queueCapacities, + parentQueueCapacities, queueCapacities.getExistingNodeLabels()); + } + + private Resource getMinResourceNormalized(String name, + Map effectiveMinRatio, Resource minResource) { + Resource ret = Resource.newInstance(minResource); + int maxLength = ResourceUtils.getNumberOfCountableResourceTypes(); + for (int i = 0; i < maxLength; i++) { + ResourceInformation nResourceInformation = + minResource.getResourceInformation(i); + + Float ratio = effectiveMinRatio.get(nResourceInformation.getName()); + if (ratio != null) { + ret.setResourceValue(i, + (long) (nResourceInformation.getValue() * ratio.floatValue())); + if (LOG.isDebugEnabled()) { + LOG.debug("Updating min resource for Queue: " + name + " as " + ret + .getResourceInformation(i) + ", Actual resource: " + + nResourceInformation.getValue() + ", ratio: " + ratio + .floatValue()); + } + } + } + return ret; + } + + private void deriveCapacityFromAbsoluteConfigurations(String label, + Resource clusterResource, ResourceCalculator rc) { + + /* + * In case when queues are configured with absolute resources, it is better + * to update capacity/max-capacity etc w.r.t absolute resource as well. In + * case of computation, these values wont be used any more. However for + * metrics and UI, its better these values are pre-computed here itself. + */ + + // 1. Update capacity as a float based on parent's minResource + float f = rc.divide(clusterResource, + queueResourceQuotas.getEffectiveMinResource(label), + parent.getQueueResourceQuotas().getEffectiveMinResource(label)); + queueCapacities.setCapacity(label, Float.isInfinite(f) ? 0 : f); + + // 2. Update max-capacity as a float based on parent's maxResource + f = rc.divide(clusterResource, + queueResourceQuotas.getEffectiveMaxResource(label), + parent.getQueueResourceQuotas().getEffectiveMaxResource(label)); + queueCapacities.setMaximumCapacity(label, Float.isInfinite(f) ? 0 : f); + + // 3. Update absolute capacity as a float based on parent's minResource and + // cluster resource. + queueCapacities.setAbsoluteCapacity(label, + queueCapacities.getCapacity(label) * parent.getQueueCapacities() + .getAbsoluteCapacity(label)); + + // 4. Update absolute max-capacity as a float based on parent's maxResource + // and cluster resource. + queueCapacities.setAbsoluteMaximumCapacity(label, + queueCapacities.getMaximumCapacity(label) * parent.getQueueCapacities() + .getAbsoluteMaximumCapacity(label)); + + // Re-visit max applications for a queue based on absolute capacity if + // needed. + if (this instanceof LeafQueue) { + LeafQueue leafQueue = (LeafQueue) this; + CapacitySchedulerConfiguration conf = csContext.getConfiguration(); + int maxApplications = conf.getMaximumApplicationsPerQueue(queuePath); + if (maxApplications < 0) { + int maxGlobalPerQueueApps = conf.getGlobalMaximumApplicationsPerQueue(); + if (maxGlobalPerQueueApps > 0) { + maxApplications = (int) (maxGlobalPerQueueApps * queueCapacities + .getAbsoluteCapacity(label)); + } else{ + maxApplications = + (int) (conf.getMaximumSystemApplications() * queueCapacities + .getAbsoluteCapacity(label)); + } + } + leafQueue.setMaxApplications(maxApplications); + + int maxApplicationsPerUser = Math.min(maxApplications, + (int) (maxApplications * (leafQueue.getUsersManager().getUserLimit() + / 100.0f) * leafQueue.getUsersManager().getUserLimitFactor())); + leafQueue.setMaxApplicationsPerUser(maxApplicationsPerUser); + LOG.info("LeafQueue:" + leafQueue.getQueuePath() + ", maxApplications=" + + maxApplications + ", maxApplicationsPerUser=" + + maxApplicationsPerUser + ", Abs Cap:" + queueCapacities + .getAbsoluteCapacity(label) + ", Cap: " + queueCapacities + .getCapacity(label) + ", MaxCap : " + queueCapacities + .getMaximumCapacity(label)); + } + } + + void updateEffectiveResources(Resource clusterResource) { + Set configuredNodelabels = + csContext.getConfiguration().getConfiguredNodeLabels(getQueuePath()); + for (String label : configuredNodelabels) { + Resource resourceByLabel = labelManager.getResourceByLabel(label, + clusterResource); + + Resource minResource = queueResourceQuotas.getConfiguredMinResource( + label); + + // Update effective resource (min/max) to each child queue. + if (getCapacityConfigType().equals( + CapacityConfigType.ABSOLUTE_RESOURCE)) { + queueResourceQuotas.setEffectiveMinResource(label, + getMinResourceNormalized(queuePath, + ((ParentQueue) parent).getEffectiveMinRatioPerResource(), + minResource)); + + // Max resource of a queue should be a minimum of {configuredMaxRes, + // parentMaxRes}. parentMaxRes could be configured value. But if not + // present could also be taken from effective max resource of parent. + Resource parentMaxRes = + parent.getQueueResourceQuotas().getConfiguredMaxResource(label); + if (parent != null && parentMaxRes.equals(Resources.none())) { + parentMaxRes = + parent.getQueueResourceQuotas().getEffectiveMaxResource(label); + } + + // Minimum of {childMaxResource, parentMaxRes}. However if + // childMaxResource is empty, consider parent's max resource alone. + Resource childMaxResource = + getQueueResourceQuotas().getConfiguredMaxResource(label); + Resource effMaxResource = Resources.min(resourceCalculator, + resourceByLabel, childMaxResource.equals(Resources.none()) ? + parentMaxRes : + childMaxResource, parentMaxRes); + queueResourceQuotas.setEffectiveMaxResource(label, + Resources.clone(effMaxResource)); + + // In cases where we still need to update some units based on + // percentage, we have to calculate percentage and update. + ResourceCalculator rc = this.csContext.getResourceCalculator(); + deriveCapacityFromAbsoluteConfigurations(label, clusterResource, rc); + } else{ + queueResourceQuotas.setEffectiveMinResource(label, Resources + .multiply(resourceByLabel, + queueCapacities.getAbsoluteCapacity(label))); + queueResourceQuotas.setEffectiveMaxResource(label, Resources + .multiply(resourceByLabel, + queueCapacities.getAbsoluteMaximumCapacity(label))); + } + + if (LOG.isDebugEnabled()) { + LOG.debug("Updating effective min resource for queue:" + queuePath + + " as effMinResource=" + queueResourceQuotas + .getEffectiveMinResource(label) + + "and Updating effective max resource as effMaxResource=" + + queueResourceQuotas.getEffectiveMaxResource(label)); + } + } + } + + public boolean isDynamicQueue() { + readLock.lock(); + + try { + return dynamicQueue; + } finally { + readLock.unlock(); + } + } + + public void setDynamicQueue(boolean dynamicQueue) { + writeLock.lock(); + + try { + this.dynamicQueue = dynamicQueue; + } finally { + writeLock.unlock(); + } + } + + public long getLastSubmittedTimestamp() { + return lastSubmittedTimestamp; + } + + // "Tab" the queue, so this queue won't be removed because of idle timeout. + public void signalToSubmitToQueue() { + writeLock.lock(); + try { + this.lastSubmittedTimestamp = System.currentTimeMillis(); + } finally { + writeLock.unlock(); + } + } } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/AbstractManagedParentQueue.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/AbstractManagedParentQueue.java index 7bdc311aea1..a9e82a6f067 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/AbstractManagedParentQueue.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/AbstractManagedParentQueue.java @@ -58,7 +58,7 @@ public void reinitialize(CSQueue newlyParsedQueue, Resource clusterResource) writeLock.lock(); try { // Set new configs - setupQueueConfigs(clusterResource); + setupQueueConfigs(clusterResource, csContext.getConfiguration()); } finally { writeLock.unlock(); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/AutoCreatedLeafQueue.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/AutoCreatedLeafQueue.java index edc5277e8a7..dd77a8088c0 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/AutoCreatedLeafQueue.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/AutoCreatedLeafQueue.java @@ -74,31 +74,15 @@ public void reinitializeFromTemplate(AutoCreatedLeafQueueConfig writeLock.lock(); try { - - this.getParent().updateClusterResource(this.csContext.getClusterResource(), - new ResourceLimits(this.csContext.getClusterResource())); - - // TODO: // reinitialize only capacities for now since 0 capacity updates // can cause // abs capacity related config computations to be incorrect if we go // through reinitialize QueueCapacities capacities = leafQueueTemplate.getQueueCapacities(); - //update abs capacities - setupConfigurableCapacities(capacities); - //reset capacities for the leaf queue mergeCapacities(capacities); - //update queue used capacity for all the node labels - CSQueueUtils.updateQueueStatistics(resourceCalculator, - csContext.getClusterResource(), - this, labelManager, null); - - //activate applications if any are pending - activateApplications(); - } finally { writeLock.unlock(); } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CSQueueUtils.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CSQueueUtils.java index 6deb7da582b..a4034768387 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CSQueueUtils.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CSQueueUtils.java @@ -22,6 +22,7 @@ import org.apache.hadoop.yarn.api.records.Resource; import org.apache.hadoop.yarn.nodelabels.CommonNodeLabelsManager; import org.apache.hadoop.yarn.server.resourcemanager.nodelabels.RMNodeLabelsManager; +import org.apache.hadoop.yarn.server.resourcemanager.placement.ApplicationPlacementContext; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.ResourceUsage; import org.apache.hadoop.yarn.server.utils.Lock; import org.apache.hadoop.yarn.util.resource.ResourceCalculator; @@ -32,7 +33,7 @@ public class CSQueueUtils { public final static float EPSILON = 0.0001f; - + /* * Used only by tests */ @@ -40,7 +41,7 @@ public static void checkMaxCapacity(String queuePath, float capacity, float maximumCapacity) { if (maximumCapacity < 0.0f || maximumCapacity > 1.0f) { throw new IllegalArgumentException( - "Illegal value of maximumCapacity " + maximumCapacity + + "Illegal value of maximumCapacity " + maximumCapacity + " used in call to setMaxCapacity for queue " + queuePath); } } @@ -58,66 +59,15 @@ public static void checkAbsoluteCapacity(String queuePath, + ")"); } } - - /** - * Check sanity of capacities: - * - capacity <= maxCapacity - * - absCapacity <= absMaximumCapacity - */ - private static void capacitiesSanityCheck(String queueName, - QueueCapacities queueCapacities) { - for (String label : queueCapacities.getExistingNodeLabels()) { - // The only thing we should care about is absolute capacity <= - // absolute max capacity otherwise the absolute max capacity is - // no longer an absolute maximum. - float absCapacity = queueCapacities.getAbsoluteCapacity(label); - float absMaxCapacity = queueCapacities.getAbsoluteMaximumCapacity(label); - if (absCapacity > absMaxCapacity) { - throw new IllegalArgumentException("Illegal queue capacity setting " - + "(abs-capacity=" + absCapacity + ") > (abs-maximum-capacity=" - + absMaxCapacity + ") for queue=[" - + queueName + "],label=[" + label + "]"); - } - } - } public static float computeAbsoluteMaximumCapacity( float maximumCapacity, CSQueue parent) { - float parentAbsMaxCapacity = + float parentAbsMaxCapacity = (parent == null) ? 1.0f : parent.getAbsoluteMaximumCapacity(); return (parentAbsMaxCapacity * maximumCapacity); } - - /** - * This method intends to be used by ReservationQueue, ReservationQueue will - * not appear in configuration file, so we shouldn't do load capacities - * settings in configuration for reservation queue. - */ - public static void updateAndCheckCapacitiesByLabel(String queuePath, - QueueCapacities queueCapacities, QueueCapacities parentQueueCapacities) { - updateAbsoluteCapacitiesByNodeLabels(queueCapacities, parentQueueCapacities); - - capacitiesSanityCheck(queuePath, queueCapacities); - } - - /** - * Do following steps for capacities - * - Load capacities from configuration - * - Update absolute capacities for new capacities - * - Check if capacities/absolute-capacities legal - */ - public static void loadUpdateAndCheckCapacities(String queuePath, - CapacitySchedulerConfiguration csConf, - QueueCapacities queueCapacities, QueueCapacities parentQueueCapacities) { - loadCapacitiesByLabelsFromConf(queuePath, - queueCapacities, csConf); - updateAbsoluteCapacitiesByNodeLabels(queueCapacities, parentQueueCapacities); - - capacitiesSanityCheck(queuePath, queueCapacities); - } - - private static void loadCapacitiesByLabelsFromConf(String queuePath, + public static void loadCapacitiesByLabelsFromConf(String queuePath, QueueCapacities queueCapacities, CapacitySchedulerConfiguration csConf) { queueCapacities.clearConfigurableFields(); Set configuredNodelabels = @@ -132,41 +82,30 @@ private static void loadCapacitiesByLabelsFromConf(String queuePath, queueCapacities.setMaxAMResourcePercentage( label, csConf.getMaximumAMResourcePercentPerPartition(queuePath, label)); - } else { + queueCapacities.setWeight(label, + csConf.getNonLabeledQueueWeight(queuePath)); + } else{ queueCapacities.setCapacity(label, csConf.getLabeledQueueCapacity(queuePath, label) / 100); queueCapacities.setMaximumCapacity(label, csConf.getLabeledQueueMaximumCapacity(queuePath, label) / 100); queueCapacities.setMaxAMResourcePercentage(label, csConf.getMaximumAMResourcePercentPerPartition(queuePath, label)); - } - } - } - - // Set absolute capacities for {capacity, maximum-capacity} - private static void updateAbsoluteCapacitiesByNodeLabels( - QueueCapacities queueCapacities, QueueCapacities parentQueueCapacities) { - for (String label : queueCapacities.getExistingNodeLabels()) { - float capacity = queueCapacities.getCapacity(label); - if (capacity > 0f) { - queueCapacities.setAbsoluteCapacity( - label, - capacity - * (parentQueueCapacities == null ? 1 : parentQueueCapacities - .getAbsoluteCapacity(label))); + queueCapacities.setWeight(label, + csConf.getLabeledQueueWeight(queuePath, label)); } - float maxCapacity = queueCapacities.getMaximumCapacity(label); - if (maxCapacity > 0f) { - queueCapacities.setAbsoluteMaximumCapacity( - label, - maxCapacity - * (parentQueueCapacities == null ? 1 : parentQueueCapacities - .getAbsoluteMaximumCapacity(label))); - } + /*float absCapacity = queueCapacities.getCapacity(label); + float absMaxCapacity = queueCapacities.getMaximumCapacity(label); + if (absCapacity > absMaxCapacity) { + throw new IllegalArgumentException("Illegal queue capacity setting " + + "(abs-capacity=" + absCapacity + ") > (abs-maximum-capacity=" + + absMaxCapacity + ") for queue=[" + + queuePath + "],label=[" + label + "]"); + }*/ } } - + /** * Update partitioned resource usage, if nodePartition == null, will update * used resource for all partitions of this queue. @@ -344,4 +283,45 @@ public static void updateConfiguredCapacityMetrics(ResourceCalculator rc, queue.getQueueCapacities().getMaximumCapacity(partition), queue.getQueueCapacities().getAbsoluteMaximumCapacity(partition)); } + + public static void updateAbsoluteCapacitiesByNodeLabels(QueueCapacities queueCapacities, + QueueCapacities parentQueueCapacities, + Set nodeLabels) { + for (String label : nodeLabels) { + // Weight will be normalized to queue.weight = + // queue.weight(sum({sibling-queues.weight})) + // When weight is set, capacity will be set to 0; + // When capacity is set, weight will be normalized to 0, + // So get larger from normalized_weight and capacity will make sure we do + // calculation correct + float capacity = Math.max( + queueCapacities.getCapacity(label), + queueCapacities + .getNormalizedWeight(label)); + if (capacity > 0f) { + queueCapacities.setAbsoluteCapacity(label, capacity * ( + parentQueueCapacities == null ? 1 : + parentQueueCapacities.getAbsoluteCapacity(label))); + } + + float maxCapacity = queueCapacities + .getMaximumCapacity(label); + if (maxCapacity > 0f) { + queueCapacities.setAbsoluteMaximumCapacity(label, maxCapacity * ( + parentQueueCapacities == null ? 1 : + parentQueueCapacities.getAbsoluteMaximumCapacity(label))); + } + } + } + + public static ApplicationPlacementContext extractQueuePath(String queuePath) { + int parentQueueNameEndIndex = queuePath.lastIndexOf("."); + if (parentQueueNameEndIndex > -1) { + String parent = queuePath.substring(0, parentQueueNameEndIndex).trim(); + String leaf = queuePath.substring(parentQueueNameEndIndex + 1).trim(); + return new ApplicationPlacementContext(leaf, parent); + } else{ + return new ApplicationPlacementContext(queuePath); + } + } } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacityScheduler.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacityScheduler.java index 86f3023a221..5b099899579 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacityScheduler.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacityScheduler.java @@ -165,6 +165,7 @@ import org.apache.hadoop.thirdparty.com.google.common.base.Preconditions; import org.apache.hadoop.thirdparty.com.google.common.util.concurrent.SettableFuture; +import static org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacitySchedulerConfiguration.DOT; import static org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacitySchedulerConfiguration.QUEUE_MAPPING; @LimitedPrivate("yarn") @@ -230,6 +231,8 @@ public Configuration getConf() { private AppPriorityACLsManager appPriorityACLManager; private boolean multiNodePlacementEnabled; + private CapacitySchedulerAutoQueueHandler autoQueueHandler; + private static boolean printedVerboseLoggingForAsyncScheduling = false; /** @@ -340,6 +343,9 @@ void initScheduler(Configuration configuration) throws this.labelManager, this.appPriorityACLManager); this.queueManager.setCapacitySchedulerContext(this); + this.autoQueueHandler = new CapacitySchedulerAutoQueueHandler( + this.queueManager, this.conf); + this.workflowPriorityMappingsMgr = new WorkflowPriorityMappingsManager(); this.activitiesManager = new ActivitiesManager(rmContext); @@ -924,7 +930,7 @@ private void addApplicationOnRecovery(ApplicationId applicationId, private CSQueue getOrCreateQueueFromPlacementContext(ApplicationId applicationId, String user, String queueName, ApplicationPlacementContext placementContext, - boolean isRecovery) { + boolean isRecovery) { CSQueue queue = getQueue(queueName); @@ -3329,44 +3335,6 @@ public MutableConfigurationProvider getMutableConfProvider() { return null; } - private LeafQueue autoCreateLeafQueue( - ApplicationPlacementContext placementContext) - throws IOException, YarnException { - - AutoCreatedLeafQueue autoCreatedLeafQueue = null; - - String leafQueueName = placementContext.getQueue(); - String parentQueueName = placementContext.getParentQueue(); - - if (!StringUtils.isEmpty(parentQueueName)) { - CSQueue parentQueue = getQueue(parentQueueName); - - if (parentQueue != null && conf.isAutoCreateChildQueueEnabled( - parentQueue.getQueuePath())) { - - ManagedParentQueue autoCreateEnabledParentQueue = - (ManagedParentQueue) parentQueue; - autoCreatedLeafQueue = new AutoCreatedLeafQueue(this, leafQueueName, - autoCreateEnabledParentQueue); - - addQueue(autoCreatedLeafQueue); - - } else{ - throw new SchedulerDynamicEditException( - "Could not auto-create leaf queue for " + leafQueueName - + ". Queue mapping specifies an invalid parent queue " - + "which does not exist " - + parentQueueName); - } - } else{ - throw new SchedulerDynamicEditException( - "Could not auto-create leaf queue for " + leafQueueName - + ". Queue mapping does not specify" - + " which parent queue it needs to be created under."); - } - return autoCreatedLeafQueue; - } - @Override public void resetSchedulerMetrics() { CapacitySchedulerMetrics.destroy(); @@ -3403,4 +3371,49 @@ public void setActivitiesManagerEnabled(boolean enabled) { public void setQueueManager(CapacitySchedulerQueueManager qm) { this.queueManager = qm; } + + @VisibleForTesting + protected LeafQueue autoCreateLeafQueue( + ApplicationPlacementContext placementContext) + throws IOException, YarnException { + String leafQueueName = placementContext.getQueue(); + String parentQueueName = placementContext.getParentQueue(); + + if (!StringUtils.isEmpty(parentQueueName)) { + + if (conf.isAutoCreateChildQueueEnabled(parentQueueName)) { + CSQueue parentQueue = getQueue(parentQueueName); + + if (parentQueue != null) { + // Case 1: Handle ManagedParentQueue + AutoCreatedLeafQueue autoCreatedLeafQueue = null; + ManagedParentQueue autoCreateEnabledParentQueue = + (ManagedParentQueue) parentQueue; + autoCreatedLeafQueue = new AutoCreatedLeafQueue(this, leafQueueName, + autoCreateEnabledParentQueue); + + addQueue(autoCreatedLeafQueue); + return autoCreatedLeafQueue; + } else{ + throw new SchedulerDynamicEditException( + "Could not auto-create leaf queue for " + leafQueueName + + ". Queue mapping specifies an invalid parent queue " + + "which does not exist " + parentQueueName); + } + } else { + try { + writeLock.lock(); + LeafQueue lq = autoQueueHandler.autoCreateQueuePath(placementContext); + return lq; + } finally { + writeLock.unlock(); + } + } + } + + throw new SchedulerDynamicEditException( + "Could not auto-create leaf queue for " + leafQueueName + + ". Queue mapping does not specify" + + " which parent queue it needs to be created under."); + } } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacitySchedulerAutoQueueHandler.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacitySchedulerAutoQueueHandler.java new file mode 100644 index 00000000000..48cfbca3788 --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacitySchedulerAutoQueueHandler.java @@ -0,0 +1,152 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + *

+ * http://www.apache.org/licenses/LICENSE-2.0 + *

+ * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity; + +import org.apache.hadoop.yarn.exceptions.YarnException; +import org.apache.hadoop.yarn.server.resourcemanager.placement.ApplicationPlacementContext; +import org.apache.hadoop.yarn.server.resourcemanager.scheduler.SchedulerDynamicEditException; + +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; + +/** + * Manages the validation and the creation of a Capacity Scheduler + * queue at runtime. + */ +public class CapacitySchedulerAutoQueueHandler { + private final CapacitySchedulerQueueManager queueManager; + private final CapacitySchedulerConfiguration conf; + private static final int MAXIMUM_DEPTH_ALLOWED = 2; + + public CapacitySchedulerAutoQueueHandler( + CapacitySchedulerQueueManager queueManager, + CapacitySchedulerConfiguration conf) { + this.queueManager = queueManager; + this.conf = conf; + } + + /** + * Creates a LeafQueue and its upper hierarchy given a path. A parent is + * eligible for creation if either the placement context creation flags are + * set, or the auto queue creation is enabled for the first static parent in + * the hierarchy. + * + * @param queue the application placement information of the queue + * @return LeafQueue part of a given queue path + * @throws YarnException if the given path is not eligible to be auto created + */ + public LeafQueue autoCreateQueuePath(ApplicationPlacementContext queue) + throws YarnException { + CSQueue parent = autoCreateParentHierarchy(queue); + + if (parent instanceof ParentQueue) { + ParentQueue parentQueue = (ParentQueue) parent; + LeafQueue leafQueue = (LeafQueue) parentQueue.addDynamicChildQueue( + queue.getFullQueuePath(), true); + queueManager.addQueue(leafQueue.getQueuePath(), leafQueue); + + return leafQueue; + } else { + throw new SchedulerDynamicEditException( + "Could not auto-create leaf queue for " + queue.getQueue() + + ". Queue mapping specifies an invalid parent queue " + + "which does not exist" + + queue.getParentQueue()); + } + } + + private CSQueue autoCreateParentHierarchy( + ApplicationPlacementContext queue) + throws YarnException { + ApplicationPlacementContext parentContext = + CSQueueUtils.extractQueuePath(queue.getParentQueue()); + List parentsToCreate = new ArrayList<>(); + + ApplicationPlacementContext queueCandidateContext = parentContext; + CSQueue existingQueueCandidate = getQueue(queueCandidateContext.getQueue()); + + while (existingQueueCandidate == null) { + parentsToCreate.add(queueCandidateContext); + queueCandidateContext = CSQueueUtils.extractQueuePath( + queueCandidateContext.getParentQueue()); + existingQueueCandidate = getQueue(queueCandidateContext.getQueue()); + } + + // Reverse the collection to to represent the hierarchy to be created + // from highest to lowest level + Collections.reverse(parentsToCreate); + + CSQueue existingParentQueue = existingQueueCandidate; + if (!(existingParentQueue instanceof ParentQueue)) { + throw new SchedulerDynamicEditException( + "Could not auto create hierarchy of " + + queue.getFullQueuePath() + ". Queue " + + existingParentQueue.getQueuePath() + + " is not a ParentQueue." + ); + } + int depthLimit = extractDepthLimit(queue, existingParentQueue); + // The number of levels to be created including the LeafQueue + // (which is last) + int levelsToCreate = parentsToCreate.size() + 1; + + if (depthLimit == 0) { + throw new SchedulerDynamicEditException("Auto creation of queue " + + queue.getFullQueuePath() + " is not enabled under parent " + + existingParentQueue.getQueuePath()); + } + + if (levelsToCreate > depthLimit) { + throw new SchedulerDynamicEditException( + "Could not auto create queue " + queue.getFullQueuePath() + + ". In order to create the desired queue hierarchy, " + + levelsToCreate + " levels of queues would need " + + "to be created, which is above the limit."); + } + + for (ApplicationPlacementContext current : parentsToCreate) { + ParentQueue parentQueue = (ParentQueue) existingParentQueue; + existingParentQueue = parentQueue.addDynamicChildQueue( + current.getFullQueuePath(), false); + queueManager.addQueue(existingParentQueue.getQueuePath(), + existingParentQueue); + } + + return existingParentQueue; + } + + private int extractDepthLimit( + ApplicationPlacementContext queue, CSQueue parentQueue) { + if ((queue.isCreateParentQueue() && queue.isCreateLeafQueue()) || + isAutoQueueCreationEnabled(parentQueue)) { + return MAXIMUM_DEPTH_ALLOWED; + } else { + return 0; + } + } + + private boolean isAutoQueueCreationEnabled(CSQueue existingParentQueue) { + return ((AbstractCSQueue) existingParentQueue).isDynamicQueue() || + conf.isAutoQueueCreationEnabled(existingParentQueue.getQueuePath()); + } + + private CSQueue getQueue(String queue) { + return queue != null ? queueManager.getQueue(queue) : null; + } +} diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacitySchedulerConfigValidator.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacitySchedulerConfigValidator.java index c3b4df4efdf..ef9f97aee18 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacitySchedulerConfigValidator.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacitySchedulerConfigValidator.java @@ -106,6 +106,10 @@ public static void validateVCores(Configuration conf) { } } + private static boolean isDynamicQueue(CSQueue csQueue) { + return ((AbstractCSQueue)csQueue).isDynamicQueue(); + } + /** * Ensure all existing queues are present. Queues cannot be deleted if its not * in Stopped state, Queue's cannot be moved from one hierarchy to other also. @@ -144,10 +148,12 @@ public static void validateQueueHierarchy( LOG.info("Deleting Queue " + queuePath + ", as it is not" + " present in the modified capacity configuration xml"); } else { - throw new IOException(oldQueue.getQueuePath() + " cannot be" - + " deleted from the capacity scheduler configuration, as the" - + " queue is not yet in stopped state. Current State : " - + oldQueue.getState()); + if (!isDynamicQueue(oldQueue)) { + throw new IOException(oldQueue.getQueuePath() + " cannot be" + + " deleted from the capacity scheduler configuration, as the" + + " queue is not yet in stopped state. Current State : " + + oldQueue.getState()); + } } } else if (!oldQueue.getQueuePath().equals(newQueue.getQueuePath())) { //Queue's cannot be moved from one hierarchy to other diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacitySchedulerConfiguration.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacitySchedulerConfiguration.java index d0ee25df300..f503074048b 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacitySchedulerConfiguration.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacitySchedulerConfiguration.java @@ -18,6 +18,7 @@ package org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity; +import org.apache.hadoop.ipc.WeightedTimeCostProvider; import org.apache.hadoop.thirdparty.com.google.common.annotations.VisibleForTesting; import org.apache.hadoop.thirdparty.com.google.common.base.Strings; import org.apache.hadoop.thirdparty.com.google.common.collect.ImmutableSet; @@ -385,6 +386,8 @@ public static final Pattern RESOURCE_PATTERN = Pattern.compile(PATTERN_FOR_ABSOLUTE_RESOURCE); + private static final String WEIGHT_SUFFIX = "w"; + public static final String MAX_PARALLEL_APPLICATIONS = "max-parallel-apps"; public static final int DEFAULT_MAX_PARALLEL_APPLICATIONS = Integer.MAX_VALUE; @@ -491,12 +494,45 @@ public void setMaximumApplicationMasterResourcePerQueuePercent(String queue, float percent) { setFloat(getQueuePrefix(queue) + MAXIMUM_AM_RESOURCE_SUFFIX, percent); } - + + private void throwExceptionForUnexpectedWeight(float weight, String queue, + String label) { + if ((weight < -1e-6 && Math.abs(weight + 1) > 1e-6) || weight > 10000) { + throw new IllegalArgumentException( + "Illegal " + "weight=" + weight + " for queue=" + queue + "label=" + + label + + ". Acceptable values: [0, 10000], -1 is same as not set"); + } + } + + public float getNonLabeledQueueWeight(String queue) { + String configuredValue = get(getQueuePrefix(queue) + CAPACITY); + float weight = extractFloatValueFromWeightConfig(configuredValue); + throwExceptionForUnexpectedWeight(weight, queue, ""); + return weight; + } + + public void setNonLabeledQueueWeight(String queue, float weight) { + set(getQueuePrefix(queue) + CAPACITY, weight + WEIGHT_SUFFIX); + } + + public void setLabeledQueueWeight(String queue, String label, float weight) { + set(getNodeLabelPrefix(queue, label) + CAPACITY, weight + WEIGHT_SUFFIX); + } + + public float getLabeledQueueWeight(String queue, String label) { + String configuredValue = get(getNodeLabelPrefix(queue, label) + CAPACITY); + float weight = extractFloatValueFromWeightConfig(configuredValue); + throwExceptionForUnexpectedWeight(weight, queue, label); + return weight; + } + public float getNonLabeledQueueCapacity(String queue) { String configuredCapacity = get(getQueuePrefix(queue) + CAPACITY); - boolean matcher = (configuredCapacity != null) + boolean absoluteResourceConfigured = (configuredCapacity != null) && RESOURCE_PATTERN.matcher(configuredCapacity).find(); - if (matcher) { + if (absoluteResourceConfigured || configuredWeightAsCapacity( + configuredCapacity)) { // Return capacity in percentage as 0 for non-root queues and 100 for // root.From AbstractCSQueue, absolute resource will be parsed and // updated. Once nodes are added/removed in cluster, capacity in @@ -729,31 +765,51 @@ public void setAccessibleNodeLabels(String queue, Set labels) { } return Collections.unmodifiableSet(set); } - - private float internalGetLabeledQueueCapacity(String queue, String label, String suffix, - float defaultValue) { + + private boolean configuredWeightAsCapacity(String configureValue) { + if (configureValue == null) { + return false; + } + return configureValue.endsWith(WEIGHT_SUFFIX); + } + + private float extractFloatValueFromWeightConfig(String configureValue) { + if (!configuredWeightAsCapacity(configureValue)) { + return -1f; + } else { + return Float.valueOf( + configureValue.substring(0, configureValue.indexOf(WEIGHT_SUFFIX))); + } + } + + private float internalGetLabeledQueueCapacity(String queue, String label, + String suffix, float defaultValue) { String capacityPropertyName = getNodeLabelPrefix(queue, label) + suffix; String configuredCapacity = get(capacityPropertyName); - boolean matcher = (configuredCapacity != null) - && RESOURCE_PATTERN.matcher(configuredCapacity).find(); - if (matcher) { + boolean absoluteResourceConfigured = + (configuredCapacity != null) && RESOURCE_PATTERN.matcher( + configuredCapacity).find(); + if (absoluteResourceConfigured || configuredWeightAsCapacity( + configuredCapacity)) { // Return capacity in percentage as 0 for non-root queues and 100 for - // root.From AbstractCSQueue, absolute resource will be parsed and - // updated. Once nodes are added/removed in cluster, capacity in - // percentage will also be re-calculated. + // root.From AbstractCSQueue, absolute resource, and weight will be parsed + // and updated separately. Once nodes are added/removed in cluster, + // capacity is percentage will also be re-calculated. return defaultValue; } float capacity = getFloat(capacityPropertyName, defaultValue); if (capacity < MINIMUM_CAPACITY_VALUE || capacity > MAXIMUM_CAPACITY_VALUE) { - throw new IllegalArgumentException("Illegal capacity of " + capacity - + " for node-label=" + label + " in queue=" + queue - + ", valid capacity should in range of [0, 100]."); + throw new IllegalArgumentException( + "Illegal capacity of " + capacity + " for node-label=" + label + + " in queue=" + queue + + ", valid capacity should in range of [0, 100]."); } if (LOG.isDebugEnabled()) { - LOG.debug("CSConf - getCapacityOfLabel: prefix=" - + getNodeLabelPrefix(queue, label) + ", capacity=" + capacity); + LOG.debug( + "CSConf - getCapacityOfLabel: prefix=" + getNodeLabelPrefix(queue, + label) + ", capacity=" + capacity); } return capacity; } @@ -1952,6 +2008,17 @@ public void setDefaultLifetimePerQueue(String queue, long defaultLifetime) { public static final String AUTO_CREATE_CHILD_QUEUE_ENABLED = AUTO_CREATE_CHILD_QUEUE_PREFIX + "enabled"; + @Private + private static final String AUTO_QUEUE_CREATION_PREFIX = + "auto-queue-creation"; + + @Private + public static final String AUTO_QUEUE_CREATION_ENABLED = + AUTO_QUEUE_CREATION_PREFIX + ".enabled"; + + @Private + public static final boolean DEFAULT_AUTO_QUEUE_CREATION_ENABLED = false; + @Private public static final String AUTO_CREATED_LEAF_QUEUE_TEMPLATE_PREFIX = "leaf-queue-template"; @@ -1988,6 +2055,20 @@ public void setAutoCreateChildQueueEnabled(String queuePath, autoCreationEnabled); } + public void setAutoQueueCreationEnabled(String queuePath, + boolean autoQueueCreation) { + setBoolean( + getQueuePrefix(queuePath) + AUTO_QUEUE_CREATION_ENABLED, + autoQueueCreation); + } + + public boolean isAutoQueueCreationEnabled(String queuePath) { + boolean isAutoQueueCreation = getBoolean( + getQueuePrefix(queuePath) + AUTO_QUEUE_CREATION_ENABLED, + DEFAULT_AUTO_QUEUE_CREATION_ENABLED); + return isAutoQueueCreation; + } + /** * Get the auto created leaf queue's template configuration prefix * Leaf queue's template capacities are configured at the parent queue diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacitySchedulerQueueManager.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacitySchedulerQueueManager.java index a44929beed6..c5ce700eef5 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacitySchedulerQueueManager.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacitySchedulerQueueManager.java @@ -20,11 +20,15 @@ import java.io.IOException; import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collections; import java.util.HashMap; import java.util.List; import java.util.Map; import java.util.Set; +import org.apache.hadoop.yarn.util.resource.ResourceUtils; +import org.apache.hadoop.yarn.util.resource.Resources; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.apache.hadoop.classification.InterfaceAudience.Private; @@ -164,6 +168,8 @@ public void initializeQueues(CapacitySchedulerConfiguration conf) setQueueAcls(authorizer, appPriorityACLManager, queues); labelManager.reinitializeQueueLabels(getQueueToLabels()); this.queueStateManager.initialize(this); + root.updateClusterResource(csContext.getClusterResource(), + new ResourceLimits(csContext.getClusterResource())); LOG.info("Initialized root queue " + root); } @@ -172,7 +178,7 @@ public void reinitializeQueues(CapacitySchedulerConfiguration newConf) throws IOException { // Parse new queues CSQueueStore newQueues = new CSQueueStore(); - CSQueue newRoot = parseQueue(this.csContext, newConf, null, + CSQueue newRoot = parseQueue(this.csContext, newConf, null, CapacitySchedulerConfiguration.ROOT, newQueues, queues, NOOP); // When failing over, if using configuration store, don't validate queue @@ -208,7 +214,7 @@ public void reinitializeQueues(CapacitySchedulerConfiguration newConf) * @param conf the CapacitySchedulerConfiguration * @param parent the parent queue * @param queueName the queue name - * @param queues all the queues + * @param newQueues all the queues * @param oldQueues the old queues * @param hook the queue hook * @return the CSQueue @@ -218,18 +224,28 @@ static CSQueue parseQueue( CapacitySchedulerContext csContext, CapacitySchedulerConfiguration conf, CSQueue parent, String queueName, - CSQueueStore queues, + CSQueueStore newQueues, CSQueueStore oldQueues, QueueHook hook) throws IOException { CSQueue queue; String fullQueueName = (parent == null) ? queueName : (parent.getQueuePath() + "." + queueName); - String[] childQueueNames = conf.getQueues(fullQueueName); + String[] staticChildQueueNames = conf.getQueues(fullQueueName); + List childQueueNames = staticChildQueueNames != null ? + Arrays.asList(staticChildQueueNames) : Collections.emptyList(); + boolean isReservableQueue = conf.isReservable(fullQueueName); boolean isAutoCreateEnabled = conf.isAutoCreateChildQueueEnabled( fullQueueName); - if (childQueueNames == null || childQueueNames.length == 0) { + boolean isDynamicParent = false; + + CSQueue oldQueue = oldQueues.get(fullQueueName); + if (oldQueue instanceof ParentQueue) { + isDynamicParent = ((ParentQueue) oldQueue).isDynamicQueue(); + } + + if (childQueueNames.size() == 0 && !isDynamicParent) { if (null == parent) { throw new IllegalStateException( "Queue configuration missing child queue names for " + queueName); @@ -254,7 +270,7 @@ static CSQueue parseQueue( } childQueues.add(resQueue); ((PlanQueue) queue).setChildQueues(childQueues); - queues.add(resQueue); + newQueues.add(resQueue); } else if (isAutoCreateEnabled) { queue = new ManagedParentQueue(csContext, queueName, parent, @@ -287,14 +303,14 @@ static CSQueue parseQueue( List childQueues = new ArrayList<>(); for (String childQueueName : childQueueNames) { CSQueue childQueue = parseQueue(csContext, conf, queue, childQueueName, - queues, oldQueues, hook); + newQueues, oldQueues, hook); childQueues.add(childQueue); } parentQueue.setChildQueues(childQueues); } - queues.add(queue); + newQueues.add(queue); LOG.info("Initialized queue: " + fullQueueName); return queue; @@ -316,11 +332,12 @@ private void updateQueues(CSQueueStore existingQueues, } } - for (CSQueue queue: existingQueues.getQueues()) { - if (newQueues.get(queue.getQueuePath()) == null && !( + for (CSQueue queue : existingQueues.getQueues()) { + if (!((AbstractCSQueue) queue).isDynamicQueue() && newQueues.get( + queue.getQueuePath()) == null && !( queue instanceof AutoCreatedLeafQueue && conf .isAutoCreateChildQueueEnabled( - queue.getParent().getQueuePath()))) { + queue.getParent().getQueuePath()))) { existingQueues.remove(queue); } } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/LeafQueue.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/LeafQueue.java index 96d309c547e..15c321fca0b 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/LeafQueue.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/LeafQueue.java @@ -168,11 +168,6 @@ public LeafQueue(CapacitySchedulerContext cs, } - protected void setupQueueConfigs(Resource clusterResource) - throws IOException { - setupQueueConfigs(clusterResource, csContext.getConfiguration()); - } - @SuppressWarnings("checkstyle:nowhitespaceafter") protected void setupQueueConfigs(Resource clusterResource, CapacitySchedulerConfiguration conf) throws @@ -200,26 +195,18 @@ protected void setupQueueConfigs(Resource clusterResource, usersManager.setUserLimit(conf.getUserLimit(getQueuePath())); usersManager.setUserLimitFactor(conf.getUserLimitFactor(getQueuePath())); + maxAMResourcePerQueuePercent = + conf.getMaximumApplicationMasterResourcePerQueuePercent( + getQueuePath()); + maxApplications = conf.getMaximumApplicationsPerQueue(getQueuePath()); if (maxApplications < 0) { - int maxGlobalPerQueueApps = schedConf - .getGlobalMaximumApplicationsPerQueue(); + int maxGlobalPerQueueApps = + csContext.getConfiguration().getGlobalMaximumApplicationsPerQueue(); if (maxGlobalPerQueueApps > 0) { maxApplications = maxGlobalPerQueueApps; - } else { - int maxSystemApps = schedConf. - getMaximumSystemApplications(); - maxApplications = - (int) (maxSystemApps * queueCapacities.getAbsoluteCapacity()); } } - maxApplicationsPerUser = Math.min(maxApplications, - (int) (maxApplications * (usersManager.getUserLimit() / 100.0f) - * usersManager.getUserLimitFactor())); - - maxAMResourcePerQueuePercent = - conf.getMaximumApplicationMasterResourcePerQueuePercent( - getQueuePath()); priorityAcls = conf.getPriorityAcls(getQueuePath(), scheduler.getMaxClusterLevelAppPriority()); @@ -537,6 +524,13 @@ protected void reinitialize( writeLock.lock(); try { + // We skip reinitialize for dynamic queues, when this is called, and + // new queue is different from this queue, we will make this queue to be + // static queue. + if (newlyParsedQueue != this) { + this.setDynamicQueue(false); + } + // Sanity check if (!(newlyParsedQueue instanceof LeafQueue) || !newlyParsedQueue .getQueuePath().equals(getQueuePath())) { @@ -560,11 +554,6 @@ protected void reinitialize( } setupQueueConfigs(clusterResource, configuration); - - // queue metrics are updated, more resource may be available - // activate the pending applications if possible - activateApplications(); - } finally { writeLock.unlock(); } @@ -639,7 +628,8 @@ public void validateSubmitApplication(ApplicationId applicationId, } // Check submission limits for queues - if (getNumApplications() >= getMaxApplications()) { + //TODO recalculate max applications because they can depend on capacity + if (getNumApplications() >= getMaxApplications() && !(this instanceof AutoCreatedLeafQueue)) { String msg = "Queue " + getQueuePath() + " already has " + getNumApplications() + " applications," @@ -650,7 +640,8 @@ public void validateSubmitApplication(ApplicationId applicationId, // Check submission limits for the user on this queue User user = usersManager.getUserAndAddIfAbsent(userName); - if (user.getTotalApplications() >= getMaxApplicationsPerUser()) { + //TODO recalculate max applications because they can depend on capacity + if (user.getTotalApplications() >= getMaxApplicationsPerUser() && !(this instanceof AutoCreatedLeafQueue)) { String msg = "Queue " + getQueuePath() + " already has " + user .getTotalApplications() + " applications from user " + userName + " cannot accept submission of application: " + applicationId; @@ -1893,14 +1884,36 @@ private void updateCurrentResourceLimits( currentResourceLimits.getLimit())); } + private void updateAbsoluteCapacitiesAndRelatedFields() { + updateAbsoluteCapacities(); + CapacitySchedulerConfiguration schedulerConf = csContext.getConfiguration(); + + // If maxApplications not set, use the system total max app, apply newly + // calculated abs capacity of the queue. + if (maxApplications <= 0) { + int maxSystemApps = schedulerConf. + getMaximumSystemApplications(); + maxApplications = + (int) (maxSystemApps * queueCapacities.getAbsoluteCapacity()); + } + maxApplicationsPerUser = Math.min(maxApplications, + (int) (maxApplications * (usersManager.getUserLimit() / 100.0f) + * usersManager.getUserLimitFactor())); + } + @Override public void updateClusterResource(Resource clusterResource, ResourceLimits currentResourceLimits) { writeLock.lock(); try { - updateCurrentResourceLimits(currentResourceLimits, clusterResource); lastClusterResource = clusterResource; + updateAbsoluteCapacitiesAndRelatedFields(); + + super.updateEffectiveResources(clusterResource); + + updateCurrentResourceLimits(currentResourceLimits, clusterResource); + // Update headroom info based on new cluster resource value // absoluteMaxCapacity now, will be replaced with absoluteMaxAvailCapacity // during allocation diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/ManagedParentQueue.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/ManagedParentQueue.java index 3ecfef462a9..88fae00f1b4 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/ManagedParentQueue.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/ManagedParentQueue.java @@ -20,9 +20,11 @@ import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.yarn.api.records.Resource; import org.apache.hadoop.yarn.exceptions.YarnException; +import org.apache.hadoop.yarn.server.resourcemanager.scheduler.ResourceLimits; import org.apache.hadoop.yarn.server.resourcemanager.scheduler .SchedulerDynamicEditException; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.AbstractCSQueue.CapacityConfigType; +import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.queuemanagement.GuaranteedOrZeroCapacityOverTimePolicy; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.common.fica .FiCaSchedulerApp; import org.apache.hadoop.yarn.util.resource.Resources; @@ -180,9 +182,10 @@ private void reinitializeQueueManagementPolicy() throws IOException { //Load template capacities QueueCapacities queueCapacities = new QueueCapacities(false); - CSQueueUtils.loadUpdateAndCheckCapacities(csContext.getConfiguration() + CSQueueUtils.loadCapacitiesByLabelsFromConf(csContext.getConfiguration() .getAutoCreatedQueueTemplateConfPrefix(getQueuePath()), - csContext.getConfiguration(), queueCapacities, getQueueCapacities()); + queueCapacities, + csContext.getConfiguration()); /** @@ -266,6 +269,11 @@ public void addChildQueue(CSQueue childQueue) ManagedParentQueue parentQueue = (ManagedParentQueue) childQueue.getParent(); + if (parentQueue == null) { + throw new SchedulerDynamicEditException( + "Parent Queue is null, should not add child queue!"); + } + String leafQueuePath = childQueue.getQueuePath(); int maxQueues = conf.getAutoCreatedQueuesMaxChildQueuesLimit( parentQueue.getQueuePath()); @@ -289,6 +297,9 @@ public void addChildQueue(CSQueue childQueue) } } + ((GuaranteedOrZeroCapacityOverTimePolicy) queueManagementPolicy) + .updateTemplateAbsoluteCapacities(parentQueue.getQueueCapacities()); + AutoCreatedLeafQueue leafQueue = (AutoCreatedLeafQueue) childQueue; super.addChildQueue(leafQueue); @@ -305,6 +316,11 @@ public void addChildQueue(CSQueue childQueue) queueManagementPolicy.getInitialLeafQueueConfiguration(leafQueue); leafQueue.reinitializeFromTemplate(initialLeafQueueTemplate); + + // Do one update cluster resource call to make sure all absolute resources + // effective resources are updated. + updateClusterResource(this.csContext.getClusterResource(), + new ResourceLimits(this.csContext.getClusterResource())); } finally { writeLock.unlock(); } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/ParentQueue.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/ParentQueue.java index 7d82faeeef4..25be1c85539 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/ParentQueue.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/ParentQueue.java @@ -27,7 +27,11 @@ import java.util.Map; import java.util.Set; +import org.apache.hadoop.thirdparty.com.google.common.collect.ImmutableList; +import org.apache.hadoop.thirdparty.com.google.common.collect.ImmutableMap; import org.apache.commons.lang3.StringUtils; +import org.apache.hadoop.yarn.exceptions.YarnRuntimeException; +import org.apache.hadoop.yarn.server.resourcemanager.scheduler.SchedulerDynamicEditException; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.apache.hadoop.classification.InterfaceAudience.Private; @@ -97,13 +101,26 @@ private final boolean allowZeroCapacitySum; + // effective min ratio per resource, it is used during updateClusterResource, + // leaf queue can use this to calculate effective resources. + // This field will not be edited, reference will point to a new immutable map + // after every time recalculation + private volatile Map effectiveMinRatioPerResource; + public ParentQueue(CapacitySchedulerContext cs, String queueName, CSQueue parent, CSQueue old) throws IOException { + this(cs, cs.getConfiguration(), queueName, parent, old); + } + + private ParentQueue(CapacitySchedulerContext cs, + CapacitySchedulerConfiguration csConf, String queueName, CSQueue parent, + CSQueue old) + throws IOException { super(cs, queueName, parent, old); this.scheduler = cs; this.rootQueue = (parent == null); - float rawCapacity = cs.getConfiguration().getNonLabeledQueueCapacity(getQueuePath()); + float rawCapacity = csConf.getNonLabeledQueueCapacity(getQueuePath()); if (rootQueue && (rawCapacity != CapacitySchedulerConfiguration.MAXIMUM_CAPACITY_VALUE)) { @@ -116,7 +133,7 @@ public ParentQueue(CapacitySchedulerContext cs, this.allowZeroCapacitySum = cs.getConfiguration().getAllowZeroCapacitySum(getQueuePath()); - setupQueueConfigs(cs.getClusterResource()); + setupQueueConfigs(cs.getClusterResource(), csConf); LOG.info("Initialized parent-queue " + queueName + " name=" + queueName + @@ -130,11 +147,12 @@ private String getQueueOrderingPolicyConfigName() { queueOrderingPolicy.getConfigName(); } - protected void setupQueueConfigs(Resource clusterResource) + protected void setupQueueConfigs(Resource clusterResource, + CapacitySchedulerConfiguration csConf) throws IOException { writeLock.lock(); try { - super.setupQueueConfigs(clusterResource); + super.setupQueueConfigs(clusterResource, csConf); StringBuilder aclsString = new StringBuilder(); for (Map.Entry e : acls.entrySet()) { aclsString.append(e.getKey() + ":" + e.getValue().getAclString()); @@ -149,7 +167,7 @@ protected void setupQueueConfigs(Resource clusterResource) } // Initialize queue ordering policy - queueOrderingPolicy = csContext.getConfiguration().getQueueOrderingPolicy( + queueOrderingPolicy = csConf.getQueueOrderingPolicy( getQueuePath(), parent == null ? null : ((ParentQueue) parent).getQueueOrderingPolicyConfigName()); @@ -172,117 +190,199 @@ protected void setupQueueConfigs(Resource clusterResource) private static float PRECISION = 0.0005f; // 0.05% precision - void setChildQueues(Collection childQueues) { - writeLock.lock(); - try { - // Validate - float childCapacities = 0; - Resource minResDefaultLabel = Resources.createResource(0, 0); - for (CSQueue queue : childQueues) { - childCapacities += queue.getCapacity(); - Resources.addTo(minResDefaultLabel, queue.getQueueResourceQuotas() - .getConfiguredMinResource()); - - // If any child queue is using percentage based capacity model vs parent - // queues' absolute configuration or vice versa, throw back an - // exception. - if (!queueName.equals("root") && getCapacity() != 0f - && !queue.getQueueResourceQuotas().getConfiguredMinResource() - .equals(Resources.none())) { - throw new IllegalArgumentException("Parent queue '" + getQueuePath() - + "' and child queue '" + queue.getQueuePath() - + "' should use either percentage based capacity" - + " configuration or absolute resource together."); - } - } + // Check weight configuration, throw exception when configuration is invalid + // return true when all children use weight mode. + private QueueCapacityType getCapacityConfigurationTypeForQueues( + Collection queues) throws IOException { + // Do we have ANY queue set capacity in any labels? + boolean percentageIsSet = false; - float delta = Math.abs(1.0f - childCapacities); // crude way to check - - if (allowZeroCapacitySum) { - // If we allow zero capacity for children, only fail if: - // Σ(childCapacities) != 1.0f OR Σ(childCapacities) != 0.0f - // - // Therefore, child queues either add up to 0% or 100%. - // - // Current capacity doesn't matter, because we apply this logic - // regardless of whether the current capacity is zero or not. - if (minResDefaultLabel.equals(Resources.none()) - && (delta > PRECISION && childCapacities > PRECISION)) { - LOG.error("Capacity validation check is relaxed for" - + " queue {}, but the capacity must be either 0% or 100%", - getQueuePath()); - throw new IllegalArgumentException("Illegal" + " capacity of " - + childCapacities + " for children of queue " + queueName); + // Do we have ANY queue set weight in any labels? + boolean weightIsSet = false; + + // Do we have ANY queue set absolute in any labels? + boolean absoluteMinResSet = false; + + StringBuilder diagMsg = new StringBuilder(); + + for (CSQueue queue : queues) { + for (String nodeLabel : queueCapacities.getExistingNodeLabels()) { + float capacityByLabel = queue.getQueueCapacities().getCapacity(nodeLabel); + if (capacityByLabel > 0) { + percentageIsSet = true; + } + float weightByLabel = queue.getQueueCapacities().getWeight(nodeLabel); + // By default weight is set to -1, so >= 0 is enough. + if (weightByLabel >= 0) { + weightIsSet = true; + diagMsg.append( + "{Queue=" + queue.getQueuePath() + ", label=" + nodeLabel + + " uses weight mode}. "); + } + if (!queue.getQueueResourceQuotas().getConfiguredMinResource(nodeLabel) + .equals(Resources.none())) { + absoluteMinResSet = true; + // There's a special handling: when absolute resource is configured, + // capacity will be calculated (and set) for UI/metrics purposes, so + // when asboluteMinResource is set, unset percentage + percentageIsSet = false; + diagMsg.append( + "{Queue=" + queue.getQueuePath() + ", label=" + nodeLabel + + " uses absolute mode}. "); + } + if (percentageIsSet) { + diagMsg.append( + "{Queue=" + queue.getQueuePath() + ", label=" + nodeLabel + + " uses percentage mode}. "); } - } else if ((minResDefaultLabel.equals(Resources.none()) - && (queueCapacities.getCapacity() > 0) && (delta > PRECISION)) - || ((queueCapacities.getCapacity() == 0) && (childCapacities > 0))) { - // allow capacities being set to 0, and enforce child 0 if parent is 0 - throw new IllegalArgumentException("Illegal" + " capacity of " - + childCapacities + " for children of queue " + queueName); } + } - // check label capacities - for (String nodeLabel : queueCapacities.getExistingNodeLabels()) { - float capacityByLabel = queueCapacities.getCapacity(nodeLabel); - // check children's labels - float sum = 0; - Resource minRes = Resources.createResource(0, 0); - Resource resourceByLabel = labelManager.getResourceByLabel(nodeLabel, - scheduler.getClusterResource()); - for (CSQueue queue : childQueues) { - sum += queue.getQueueCapacities().getCapacity(nodeLabel); - - // If any child queue of a label is using percentage based capacity - // model vs parent queues' absolute configuration or vice versa, throw - // back an exception - if (!queueName.equals("root") && !this.capacityConfigType - .equals(queue.getCapacityConfigType())) { - throw new IllegalArgumentException("Parent queue '" + getQueuePath() - + "' and child queue '" + queue.getQueuePath() - + "' should use either percentage based capacity" - + "configuration or absolute resource together for label:" - + nodeLabel); - } + // If we have mixed capacity, weight or absolute resource (any of the two) + // We will throw exception + // Root queue is an exception here, because by default root queue returns + // 100 as capacity no matter what. We should look into this case in the + // future. To avoid impact too many code paths, we don;t check root queue's + // config. + if (queues.iterator().hasNext() && + !queues.iterator().next().getQueuePath().equals( + CapacitySchedulerConfiguration.ROOT) && + (percentageIsSet ? 1 : 0) + (weightIsSet ? 1 : 0) + (absoluteMinResSet ? + 1 : + 0) > 1) { + throw new IOException("Parent queue '" + getQueuePath() + + "' have children queue used mixed of " + + " weight mode, percentage and absolute mode, it is not allowed, please " + + "double check, details:" + diagMsg.toString()); + } - // Accumulate all min/max resource configured for all child queues. - Resources.addTo(minRes, queue.getQueueResourceQuotas() - .getConfiguredMinResource(nodeLabel)); - } + if (weightIsSet) { + return QueueCapacityType.WEIGHT; + } else if (absoluteMinResSet) { + return QueueCapacityType.ABSOLUTE_RESOURCE; + } else if (percentageIsSet) { + return QueueCapacityType.PERCENT; + } else { + // When all values equals to 0, consider it is a percent mode. + return QueueCapacityType.PERCENT; + } + } - float labelDelta = Math.abs(1.0f - sum); - - if (allowZeroCapacitySum) { - // Similar to above, we only throw exception if - // Σ(childCapacities) != 1.0f OR Σ(childCapacities) != 0.0f - if (minResDefaultLabel.equals(Resources.none()) - && capacityByLabel > 0 - && (labelDelta > PRECISION && sum > PRECISION)) { - LOG.error("Capacity validation check is relaxed for" - + " queue {}, but the capacity must be either 0% or 100%", - getQueuePath()); - throw new IllegalArgumentException( - "Illegal" + " capacity of " + sum + " for children of queue " - + queueName + " for label=" + nodeLabel); - } - } else if ((minResDefaultLabel.equals(Resources.none()) - && capacityByLabel > 0 - && Math.abs(1.0f - sum) > PRECISION) - || (capacityByLabel == 0) && (sum > 0)) { - throw new IllegalArgumentException( - "Illegal" + " capacity of " + sum + " for children of queue " - + queueName + " for label=" + nodeLabel); + private enum QueueCapacityType { + WEIGHT, ABSOLUTE_RESOURCE, PERCENT; + } + + /** + * Set child queue and verify capacities + * +--------------+---------------------------+-------------------------------------+------------------------+ + * | | parent-weight | parent-pct | parent-abs | + * +--------------+---------------------------+-------------------------------------+------------------------+ + * | child-weight | No specific check | No specific check | X | + * +--------------+---------------------------+-------------------------------------+------------------------+ + * | child-pct | Sum(children.capacity) = | When: | X | + * | | 0 OR 100 | parent.capacity>0 | | + * | | | sum(children.capacity)=100 OR 0 | | + * | | | parent.capacity=0 | | + * | | | sum(children.capacity)=0 | | + * +--------------+---------------------------+-------------------------------------+------------------------+ + * | child-abs | X | X | Sum(children.minRes)<= | + * | | | | parent.minRes | + * +--------------+---------------------------+-------------------------------------+------------------------+ + * @param childQueues + */ + void setChildQueues(Collection childQueues) throws IOException { + writeLock.lock(); + try { + QueueCapacityType childrenCapacityType = + getCapacityConfigurationTypeForQueues(childQueues); + QueueCapacityType parentCapacityType = + getCapacityConfigurationTypeForQueues(ImmutableList.of(this)); + + if (childrenCapacityType == QueueCapacityType.ABSOLUTE_RESOURCE + || parentCapacityType == QueueCapacityType.ABSOLUTE_RESOURCE) { + // We don't allow any mixed absolute + {weight, percentage} between + // children and parent + if (childrenCapacityType != parentCapacityType && !this.getQueuePath() + .equals(CapacitySchedulerConfiguration.ROOT)) { + throw new IOException("Parent=" + this.getQueuePath() + + ": When absolute minResource is used, we must make sure both " + + "parent and child all use absolute minResource"); } // Ensure that for each parent queue: parent.min-resource >= // Σ(child.min-resource). - Resource parentMinResource = queueResourceQuotas - .getConfiguredMinResource(nodeLabel); - if (!parentMinResource.equals(Resources.none()) && Resources.lessThan( - resourceCalculator, resourceByLabel, parentMinResource, minRes)) { - throw new IllegalArgumentException("Parent Queues" + " capacity: " - + parentMinResource + " is less than" + " to its children:" - + minRes + " for queue:" + queueName); + for (String nodeLabel : queueCapacities.getExistingNodeLabels()) { + Resource minRes = Resources.createResource(0, 0); + for (CSQueue queue : childQueues) { + // Accumulate all min/max resource configured for all child queues. + Resources.addTo(minRes, queue.getQueueResourceQuotas() + .getConfiguredMinResource(nodeLabel)); + } + Resource resourceByLabel = labelManager.getResourceByLabel(nodeLabel, + scheduler.getClusterResource()); + Resource parentMinResource = + queueResourceQuotas.getConfiguredMinResource(nodeLabel); + if (!parentMinResource.equals(Resources.none()) && Resources.lessThan( + resourceCalculator, resourceByLabel, parentMinResource, minRes)) { + throw new IOException( + "Parent Queues" + " capacity: " + parentMinResource + + " is less than" + " to its children:" + minRes + + " for queue:" + queueName); + } + } + } + + // When child uses percent + if (childrenCapacityType == QueueCapacityType.PERCENT) { + float childrenPctSum = 0; + // check label capacities + for (String nodeLabel : queueCapacities.getExistingNodeLabels()) { + // check children's labels + childrenPctSum = 0; + for (CSQueue queue : childQueues) { + childrenPctSum += queue.getQueueCapacities().getCapacity(nodeLabel); + } + + if (Math.abs(1 - childrenPctSum) > PRECISION) { + // When children's percent sum != 100% + if (Math.abs(childrenPctSum) > PRECISION) { + // It is wrong when percent sum != {0, 1} + throw new IOException( + "Illegal" + " capacity sum of " + childrenPctSum + + " for children of queue " + queueName + " for label=" + + nodeLabel + ". It should be either 0 or 1.0"); + } else{ + // We also allow children's percent sum = 0 under the following + // conditions + // - Parent uses weight mode + // - Parent uses percent mode, and parent has + // (capacity=0 OR allowZero) + if (parentCapacityType == QueueCapacityType.PERCENT) { + if ((Math.abs(queueCapacities.getCapacity(nodeLabel)) + > PRECISION) && (!allowZeroCapacitySum)) { + throw new IOException( + "Illegal" + " capacity sum of " + childrenPctSum + + " for children of queue " + queueName + + " for label=" + nodeLabel + + ". It is set to 0, but parent percent != 0, and " + + "doesn't allow children capacity to set to 0"); + } + } + } + } else { + // Even if child pct sum == 1.0, we will make sure parent has + // positive percent. + if (parentCapacityType == QueueCapacityType.PERCENT && Math.abs( + queueCapacities.getCapacity(nodeLabel)) <= 0f + && !allowZeroCapacitySum) { + throw new IOException( + "Illegal" + " capacity sum of " + childrenPctSum + + " for children of queue " + queueName + " for label=" + + nodeLabel + ". queue=" + queueName + + " has zero capacity, but child" + + "queues have positive capacities"); + } + } } } @@ -373,12 +473,113 @@ public String toString() { "numApps=" + getNumApplications() + ", " + "numContainers=" + getNumContainers(); } + + private CapacitySchedulerConfiguration getConfForAutoCreatedQueue( + String childQueuePath, boolean isLeaf) { + // Copy existing config + CapacitySchedulerConfiguration dupCSConfig = new CapacitySchedulerConfiguration( + csContext.getConfiguration(), false); + if (isLeaf) { + // FIXME: Ideally we should disable user limit factor, see YARN-10531 + // dupCSConfig.setUserLimitFactor(childQueuePath, ); + + // Set Max AM percentage to a higher value + dupCSConfig.setMaximumApplicationMasterResourcePerQueuePercent( + childQueuePath, 0.5f); + } + + return dupCSConfig; + } + + private CSQueue createNewQueue(String childQueuePath, boolean isLeaf) + throws SchedulerDynamicEditException { + try { + AbstractCSQueue childQueue; + String queueShortName = childQueuePath.substring( + childQueuePath.lastIndexOf(".") + 1); + + if (isLeaf) { + childQueue = new LeafQueue(csContext, + getConfForAutoCreatedQueue(childQueuePath, isLeaf), queueShortName, + this, null); + } else{ + childQueue = new ParentQueue(csContext, + getConfForAutoCreatedQueue(childQueuePath, isLeaf), queueShortName, + this, null); + } + childQueue.setDynamicQueue(true); + // It should be sufficient now, we don't need to set more, because weights + // related setup will be handled in updateClusterResources + + return childQueue; + } catch (IOException e) { + throw new SchedulerDynamicEditException(e.toString()); + } + } + + // New method to add child queue + public CSQueue addDynamicChildQueue(String childQueuePath, boolean isLeaf) + throws SchedulerDynamicEditException { + writeLock.lock(); + try { + // Check if queue exists, if queue exists, write a warning message (this + // should not happen, since it will be handled before calling this method) + // , but we will move on. + CSQueue queue = + csContext.getCapacitySchedulerQueueManager().getQueueByFullName( + childQueuePath); + if (queue != null) { + LOG.warn( + "This should not happen, trying to create queue=" + childQueuePath + + ", however the queue already exists"); + return queue; + } + + // First, check if we allow creation or not + boolean weightsAreUsed = false; + try { + if (isLeaf && childQueues.isEmpty()) { + weightsAreUsed = true; + } else { + weightsAreUsed = getCapacityConfigurationTypeForQueues(childQueues) + == QueueCapacityType.WEIGHT; + } + } catch (IOException e) { + LOG.warn("Caught Exception during auto queue creation", e); + } + if (!weightsAreUsed) { + throw new SchedulerDynamicEditException( + "Trying to create new queue=" + childQueuePath + + " but not all the queues under parent=" + this.getQueuePath() + + " are using weight-based capacity. Failed to created queue"); + } + + CSQueue newQueue = createNewQueue(childQueuePath, isLeaf); + this.childQueues.add(newQueue); + + // Call updateClusterResource, which will deal with all effectiveMin/MaxResource + // Calculation + this.updateClusterResource(csContext.getClusterResource(), + new ResourceLimits(this.csContext.getClusterResource())); + + return newQueue; + } finally { + writeLock.unlock(); + } + } @Override public void reinitialize(CSQueue newlyParsedQueue, Resource clusterResource) throws IOException { writeLock.lock(); try { + // We skip reinitialize for dynamic queues, when this is called, and + // new queue is different from this queue, we will make this queue to be + // static queue. + if (newlyParsedQueue != this) { + this.setDynamicQueue(false); + } + // Sanity check if (!(newlyParsedQueue instanceof ParentQueue) || !newlyParsedQueue .getQueuePath().equals(getQueuePath())) { @@ -390,7 +591,7 @@ public void reinitialize(CSQueue newlyParsedQueue, ParentQueue newlyParsedParentQueue = (ParentQueue) newlyParsedQueue; // Set new configs - setupQueueConfigs(clusterResource); + setupQueueConfigs(clusterResource, csContext.getConfiguration()); // Re-configure existing child queues and add new ones // The CS has already checked to ensure all existing child queues are present! @@ -446,13 +647,16 @@ public void reinitialize(CSQueue newlyParsedQueue, Map.Entry e = itr.next(); String queueName = e.getKey(); if (!newChildQueues.containsKey(queueName)) { + if (((AbstractCSQueue)e.getValue()).isDynamicQueue()) { + // Don't remove dynamic queue if we cannot find it in the config. + continue; + } itr.remove(); } } // Re-sort all queues - childQueues.clear(); - childQueues.addAll(currentChildQueues.values()); + setChildQueues(currentChildQueues.values()); // Make sure we notifies QueueOrderingPolicy queueOrderingPolicy.setQueues(childQueues); @@ -788,14 +992,24 @@ private boolean canAssign(Resource clusterResource, FiCaSchedulerNode node) { } private ResourceLimits getResourceLimitsOfChild(CSQueue child, - Resource clusterResource, Resource parentLimits, - String nodePartition) { + Resource clusterResource, ResourceLimits parentLimits, + String nodePartition, boolean netLimit) { // Set resource-limit of a given child, child.limit = // min(my.limit - my.used + child.used, child.max) + // First, cap parent limit by parent's max + parentLimits.setLimit(Resources.min(resourceCalculator, clusterResource, + parentLimits.getLimit(), + queueResourceQuotas.getEffectiveMaxResource(nodePartition))); + // Parent available resource = parent-limit - parent-used-resource + Resource limit = parentLimits.getLimit(); + if (netLimit) { + limit = parentLimits.getNetLimit(); + } Resource parentMaxAvailableResource = Resources.subtract( - parentLimits, queueUsage.getUsed(nodePartition)); + limit, queueUsage.getUsed(nodePartition)); + // Deduct killable from used Resources.addTo(parentMaxAvailableResource, getTotalKillableResource(nodePartition)); @@ -804,15 +1018,6 @@ private ResourceLimits getResourceLimitsOfChild(CSQueue child, Resource childLimit = Resources.add(parentMaxAvailableResource, child.getQueueResourceUsage().getUsed(nodePartition)); - // Get child's max resource - Resource childConfiguredMaxResource = child - .getEffectiveMaxCapacityDown(nodePartition, minimumAllocation); - - // Child's limit should be capped by child configured max resource - childLimit = - Resources.min(resourceCalculator, clusterResource, childLimit, - childConfiguredMaxResource); - // Normalize before return childLimit = Resources.roundDown(resourceCalculator, childLimit, minimumAllocation); @@ -841,8 +1046,8 @@ private CSAssignment assignContainersToChildQueues(Resource cluster, // Get ResourceLimits of child queue before assign containers ResourceLimits childLimits = - getResourceLimitsOfChild(childQueue, cluster, limits.getNetLimit(), - candidates.getPartition()); + getResourceLimitsOfChild(childQueue, cluster, limits, + candidates.getPartition(), true); CSAssignment childAssignment = childQueue.assignContainers(cluster, candidates, childLimits, schedulingMode); @@ -941,6 +1146,55 @@ public void updateClusterResource(Resource clusterResource, ResourceLimits resourceLimits) { writeLock.lock(); try { + // Special handle root queue + if (rootQueue) { + for (String nodeLabel : queueCapacities.getExistingNodeLabels()) { + if (queueCapacities.getWeight(nodeLabel) > 0) { + queueCapacities.setNormalizedWeight(nodeLabel, 1f); + } + } + } + + // Update absolute capacities of this queue, this need to happen before + // below calculation for effective capacities + updateAbsoluteCapacities(); + + // Normalize all dynamic queue queue's weight to 1 for all accessible node + // labels, this is important because existing node labels could keep + // changing when new node added, or node label mapping changed. We need + // this to ensure auto created queue can access all labels. + for (String nodeLabel : queueCapacities.getExistingNodeLabels()) { + for (CSQueue queue : childQueues) { + // For dynamic queue, we will set weight to 1 every time, because it + // is possible new labels added to the parent. + if (((AbstractCSQueue) queue).isDynamicQueue()) { + queue.getQueueCapacities().setWeight(nodeLabel, 1f); + } + } + } + + // Normalize weight of children + if (getCapacityConfigurationTypeForQueues(childQueues) + == QueueCapacityType.WEIGHT) { + for (String nodeLabel : queueCapacities.getExistingNodeLabels()) { + float sumOfWeight = 0; + + for (CSQueue queue : childQueues) { + float weight = Math.max(0, + queue.getQueueCapacities().getWeight(nodeLabel)); + sumOfWeight += weight; + } + // When sum of weight == 0, skip setting normalized_weight (so + // normalized weight will be 0). + if (Math.abs(sumOfWeight) > 1e-6) { + for (CSQueue queue : childQueues) { + queue.getQueueCapacities().setNormalizedWeight(nodeLabel, + queue.getQueueCapacities().getWeight(nodeLabel) / sumOfWeight); + } + } + } + } + // Update effective capacity in all parent queue. Set configuredNodelabels = csContext.getConfiguration() .getConfiguredNodeLabels(getQueuePath()); @@ -952,8 +1206,8 @@ public void updateClusterResource(Resource clusterResource, for (CSQueue childQueue : childQueues) { // Get ResourceLimits of child queue before assign containers ResourceLimits childLimits = getResourceLimitsOfChild(childQueue, - clusterResource, resourceLimits.getLimit(), - RMNodeLabelsManager.NO_LABEL); + clusterResource, resourceLimits, + RMNodeLabelsManager.NO_LABEL, false); childQueue.updateClusterResource(clusterResource, childLimits); } @@ -963,6 +1217,9 @@ public void updateClusterResource(Resource clusterResource, CSQueueUtils.updateConfiguredCapacityMetrics(resourceCalculator, labelManager.getResourceByLabel(null, clusterResource), RMNodeLabelsManager.NO_LABEL, this); + } catch (IOException e) { + LOG.error("Fatal issue found: e", e); + throw new YarnRuntimeException("Fatal issue during scheduling", e); } finally { writeLock.unlock(); } @@ -979,16 +1236,13 @@ private void calculateEffectiveResourcesAndCapacity(String label, // cluster resource. Resource resourceByLabel = labelManager.getResourceByLabel(label, clusterResource); - if (getQueuePath().equals("root")) { - queueResourceQuotas.setConfiguredMinResource(label, resourceByLabel); - queueResourceQuotas.setConfiguredMaxResource(label, resourceByLabel); - queueResourceQuotas.setEffectiveMinResource(label, resourceByLabel); - queueResourceQuotas.setEffectiveMaxResource(label, resourceByLabel); - queueCapacities.setAbsoluteCapacity(label, 1.0f); - } + + /* + * == Below logic are added to calculate effectiveMinRatioPerResource == + */ // Total configured min resources of direct children of this given parent - // queue. + // queue Resource configuredMinResources = Resource.newInstance(0L, 0); for (CSQueue childQueue : getChildQueues()) { Resources.addTo(configuredMinResources, @@ -1014,90 +1268,16 @@ private void calculateEffectiveResourcesAndCapacity(String label, } } - Map effectiveMinRatioPerResource = getEffectiveMinRatioPerResource( + effectiveMinRatioPerResource = getEffectiveMinRatioPerResource( configuredMinResources, numeratorForMinRatio); - // loop and do this for all child queues - for (CSQueue childQueue : getChildQueues()) { - Resource minResource = childQueue.getQueueResourceQuotas() - .getConfiguredMinResource(label); - - // Update effective resource (min/max) to each child queue. - if (childQueue.getCapacityConfigType() - .equals(CapacityConfigType.ABSOLUTE_RESOURCE)) { - childQueue.getQueueResourceQuotas().setEffectiveMinResource(label, - getMinResourceNormalized( - childQueue.getQueuePath(), - effectiveMinRatioPerResource, - minResource)); - - // Max resource of a queue should be a minimum of {configuredMaxRes, - // parentMaxRes}. parentMaxRes could be configured value. But if not - // present could also be taken from effective max resource of parent. - Resource parentMaxRes = queueResourceQuotas - .getConfiguredMaxResource(label); - if (parent != null && parentMaxRes.equals(Resources.none())) { - parentMaxRes = parent.getQueueResourceQuotas() - .getEffectiveMaxResource(label); - } - - // Minimum of {childMaxResource, parentMaxRes}. However if - // childMaxResource is empty, consider parent's max resource alone. - Resource childMaxResource = childQueue.getQueueResourceQuotas() - .getConfiguredMaxResource(label); - Resource effMaxResource = Resources.min(resourceCalculator, - resourceByLabel, childMaxResource.equals(Resources.none()) - ? parentMaxRes - : childMaxResource, - parentMaxRes); - childQueue.getQueueResourceQuotas().setEffectiveMaxResource(label, - Resources.clone(effMaxResource)); - - // In cases where we still need to update some units based on - // percentage, we have to calculate percentage and update. - deriveCapacityFromAbsoluteConfigurations(label, clusterResource, rc, - childQueue); - } else { - childQueue.getQueueResourceQuotas().setEffectiveMinResource(label, - Resources.multiply(resourceByLabel, - childQueue.getQueueCapacities().getAbsoluteCapacity(label))); - childQueue.getQueueResourceQuotas().setEffectiveMaxResource(label, - Resources.multiply(resourceByLabel, childQueue.getQueueCapacities() - .getAbsoluteMaximumCapacity(label))); - } - - if (LOG.isDebugEnabled()) { - LOG.debug("Updating effective min resource for queue:" - + childQueue.getQueuePath() + " as effMinResource=" - + childQueue.getQueueResourceQuotas().getEffectiveMinResource(label) - + "and Updating effective max resource as effMaxResource=" - + childQueue.getQueueResourceQuotas() - .getEffectiveMaxResource(label)); - } - } - } - - private Resource getMinResourceNormalized(String name, Map effectiveMinRatio, - Resource minResource) { - Resource ret = Resource.newInstance(minResource); - int maxLength = ResourceUtils.getNumberOfCountableResourceTypes(); - for (int i = 0; i < maxLength; i++) { - ResourceInformation nResourceInformation = minResource - .getResourceInformation(i); - - Float ratio = effectiveMinRatio.get(nResourceInformation.getName()); - if (ratio != null) { - ret.setResourceValue(i, - (long) (nResourceInformation.getValue() * ratio.floatValue())); - if (LOG.isDebugEnabled()) { - LOG.debug("Updating min resource for Queue: " + name + " as " - + ret.getResourceInformation(i) + ", Actual resource: " - + nResourceInformation.getValue() + ", ratio: " - + ratio.floatValue()); - } - } + // Update effective resources for my self; + if (rootQueue) { + queueResourceQuotas.setEffectiveMinResource(label, resourceByLabel); + queueResourceQuotas.setEffectiveMaxResource(label, resourceByLabel); + } else{ + super.updateEffectiveResources(clusterResource); } - return ret; } private Map getEffectiveMinRatioPerResource( @@ -1121,74 +1301,7 @@ private Resource getMinResourceNormalized(String name, Map effect } } } - return effectiveMinRatioPerResource; - } - - private void deriveCapacityFromAbsoluteConfigurations(String label, - Resource clusterResource, ResourceCalculator rc, CSQueue childQueue) { - - /* - * In case when queues are configured with absolute resources, it is better - * to update capacity/max-capacity etc w.r.t absolute resource as well. In - * case of computation, these values wont be used any more. However for - * metrics and UI, its better these values are pre-computed here itself. - */ - - // 1. Update capacity as a float based on parent's minResource - childQueue.getQueueCapacities().setCapacity(label, - rc.divide(clusterResource, - childQueue.getQueueResourceQuotas().getEffectiveMinResource(label), - getQueueResourceQuotas().getEffectiveMinResource(label))); - - // 2. Update max-capacity as a float based on parent's maxResource - childQueue.getQueueCapacities().setMaximumCapacity(label, - rc.divide(clusterResource, - childQueue.getQueueResourceQuotas().getEffectiveMaxResource(label), - getQueueResourceQuotas().getEffectiveMaxResource(label))); - - // 3. Update absolute capacity as a float based on parent's minResource and - // cluster resource. - childQueue.getQueueCapacities().setAbsoluteCapacity(label, - childQueue.getQueueCapacities().getCapacity(label) - * getQueueCapacities().getAbsoluteCapacity(label)); - - // 4. Update absolute max-capacity as a float based on parent's maxResource - // and cluster resource. - childQueue.getQueueCapacities().setAbsoluteMaximumCapacity(label, - childQueue.getQueueCapacities().getMaximumCapacity(label) - * getQueueCapacities().getAbsoluteMaximumCapacity(label)); - - // Re-visit max applications for a queue based on absolute capacity if - // needed. - if (childQueue instanceof LeafQueue) { - LeafQueue leafQueue = (LeafQueue) childQueue; - CapacitySchedulerConfiguration conf = csContext.getConfiguration(); - int maxApplications = - conf.getMaximumApplicationsPerQueue(childQueue.getQueuePath()); - if (maxApplications < 0) { - int maxGlobalPerQueueApps = conf.getGlobalMaximumApplicationsPerQueue(); - if (maxGlobalPerQueueApps > 0) { - maxApplications = (int) (maxGlobalPerQueueApps * - childQueue.getQueueCapacities().getAbsoluteCapacity(label)); - } else { - maxApplications = (int) (conf.getMaximumSystemApplications() - * childQueue.getQueueCapacities().getAbsoluteCapacity(label)); - } - } - leafQueue.setMaxApplications(maxApplications); - - int maxApplicationsPerUser = Math.min(maxApplications, - (int) (maxApplications - * (leafQueue.getUsersManager().getUserLimit() / 100.0f) - * leafQueue.getUsersManager().getUserLimitFactor())); - leafQueue.setMaxApplicationsPerUser(maxApplicationsPerUser); - LOG.info("LeafQueue:" + leafQueue.getQueuePath() + ", maxApplications=" - + maxApplications + ", maxApplicationsPerUser=" - + maxApplicationsPerUser + ", Abs Cap:" - + childQueue.getQueueCapacities().getAbsoluteCapacity(label) + ", Cap: " - + childQueue.getQueueCapacities().getCapacity(label) + ", MaxCap : " - + childQueue.getQueueCapacities().getMaximumCapacity(label)); - } + return ImmutableMap.copyOf(effectiveMinRatioPerResource); } @Override @@ -1463,4 +1576,9 @@ void decrementRunnableApps() { writeLock.unlock(); } } + + // This is a locking free method + Map getEffectiveMinRatioPerResource() { + return effectiveMinRatioPerResource; + } } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/PlanQueue.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/PlanQueue.java index 79afcdc2aaf..4dd3317e3eb 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/PlanQueue.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/PlanQueue.java @@ -46,6 +46,7 @@ public PlanQueue(CapacitySchedulerContext cs, String queueName, CSQueue parent, CSQueue old) throws IOException { super(cs, queueName, parent, old); + updateAbsoluteCapacities(); this.schedulerContext = cs; // Set the reservation queue attributes for the Plan @@ -100,7 +101,7 @@ public void reinitialize(CSQueue newlyParsedQueue, } // Set new configs - setupQueueConfigs(clusterResource); + setupQueueConfigs(clusterResource, csContext.getConfiguration()); updateQuotas(newlyParsedParentQueue.userLimit, newlyParsedParentQueue.userLimitFactor, diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/QueueCapacities.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/QueueCapacities.java index c1b715742ce..86d35d6cdd5 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/QueueCapacities.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/QueueCapacities.java @@ -50,7 +50,7 @@ public QueueCapacities(boolean isRoot) { // Usage enum here to make implement cleaner private enum CapacityType { USED_CAP(0), ABS_USED_CAP(1), MAX_CAP(2), ABS_MAX_CAP(3), CAP(4), ABS_CAP(5), - MAX_AM_PERC(6), RESERVED_CAP(7), ABS_RESERVED_CAP(8); + MAX_AM_PERC(6), RESERVED_CAP(7), ABS_RESERVED_CAP(8), WEIGHT(9), NORMALIZED_WEIGHT(10); private int idx; @@ -64,6 +64,9 @@ private CapacityType(int idx) { public Capacities() { capacitiesArr = new float[CapacityType.values().length]; + + // Set weight to -1 by default (means not set) + capacitiesArr[CapacityType.WEIGHT.idx] = -1; } @Override @@ -74,10 +77,12 @@ public String toString() { .append("max_cap=" + capacitiesArr[2] + "%, ") .append("abs_max_cap=" + capacitiesArr[3] + "%, ") .append("cap=" + capacitiesArr[4] + "%, ") - .append("abs_cap=" + capacitiesArr[5] + "%}") - .append("max_am_perc=" + capacitiesArr[6] + "%}") - .append("reserved_cap=" + capacitiesArr[7] + "%}") - .append("abs_reserved_cap=" + capacitiesArr[8] + "%}"); + .append("abs_cap=" + capacitiesArr[5] + "%, ") + .append("max_am_perc=" + capacitiesArr[6] + "%, ") + .append("reserved_cap=" + capacitiesArr[7] + "%, ") + .append("abs_reserved_cap=" + capacitiesArr[8] + "%, ") + .append("weight=" + capacitiesArr[9] + "w, ") + .append("normalized_weight=" + capacitiesArr[10] + "w}"); return sb.toString(); } } @@ -87,6 +92,10 @@ private float _get(String label, CapacityType type) { try { Capacities cap = capacitiesMap.get(label); if (null == cap) { + // Special handle weight mode + if (type == CapacityType.WEIGHT) { + return -1f; + } return LABEL_DOESNT_EXIST_CAP; } return cap.capacitiesArr[type.idx]; @@ -270,6 +279,40 @@ public void setAbsoluteReservedCapacity(String label, float value) { _set(label, CapacityType.ABS_RESERVED_CAP, value); } + /* Weight Getter and Setter */ + public float getWeight() { + return _get(NL, CapacityType.WEIGHT); + } + + public float getWeight(String label) { + return _get(label, CapacityType.WEIGHT); + } + + public void setWeight(float value) { + _set(NL, CapacityType.WEIGHT, value); + } + + public void setWeight(String label, float value) { + _set(label, CapacityType.WEIGHT, value); + } + + /* Weight Getter and Setter */ + public float getNormalizedWeight() { + return _get(NL, CapacityType.NORMALIZED_WEIGHT); + } + + public float getNormalizedWeight(String label) { + return _get(label, CapacityType.NORMALIZED_WEIGHT); + } + + public void setNormalizedWeight(float value) { + _set(NL, CapacityType.NORMALIZED_WEIGHT, value); + } + + public void setNormalizedWeight(String label, float value) { + _set(label, CapacityType.NORMALIZED_WEIGHT, value); + } + /** * Clear configurable fields, like * (absolute)capacity/(absolute)maximum-capacity, this will be used by queue @@ -284,6 +327,7 @@ public void clearConfigurableFields() { _set(label, CapacityType.MAX_CAP, 0); _set(label, CapacityType.ABS_CAP, 0); _set(label, CapacityType.ABS_MAX_CAP, 0); + _set(label, CapacityType.WEIGHT, 0); } } finally { writeLock.unlock(); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/ReservationQueue.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/ReservationQueue.java index d59c02bc655..ebac4c20b67 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/ReservationQueue.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/ReservationQueue.java @@ -22,8 +22,6 @@ import org.apache.hadoop.yarn.api.records.Resource; import org.apache.hadoop.yarn.server.resourcemanager.reservation.ReservationSystem; -import org.apache.hadoop.yarn.server.resourcemanager.scheduler.SchedulerDynamicEditException; -import org.apache.hadoop.yarn.server.resourcemanager.scheduler.common.QueueEntitlement; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -86,6 +84,6 @@ private void updateQuotas(int userLimit, float userLimitFactor, @Override protected void setupConfigurableCapacities(CapacitySchedulerConfiguration configuration) { - super.setupConfigurableCapacities(queueCapacities); + super.updateAbsoluteCapacities(); } } \ No newline at end of file diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/queuemanagement/GuaranteedOrZeroCapacityOverTimePolicy.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/queuemanagement/GuaranteedOrZeroCapacityOverTimePolicy.java index 90cbf4be27e..ab99317888d 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/queuemanagement/GuaranteedOrZeroCapacityOverTimePolicy.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/queuemanagement/GuaranteedOrZeroCapacityOverTimePolicy.java @@ -19,6 +19,7 @@ .queuemanagement; import org.apache.hadoop.thirdparty.com.google.common.annotations.VisibleForTesting; +import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CSQueueUtils; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity .QueueManagementDynamicEditPolicy; import org.slf4j.Logger; @@ -358,6 +359,12 @@ private void initializeLeafQueueTemplate(ManagedParentQueue parentQueue) public List computeQueueManagementChanges() throws SchedulerDynamicEditException { + // Update template absolute capacities as the capacities could have changed + // in weight mode + updateTemplateAbsoluteCapacities(managedParentQueue.getQueueCapacities(), + (GuaranteedOrZeroCapacityOverTimePolicy) + managedParentQueue.getAutoCreatedQueueManagementPolicy()); + //TODO : Add support for node labels on leaf queue template configurations //synch/add missing leaf queue(s) if any to state updateLeafQueueState(); @@ -470,6 +477,24 @@ private void initializeLeafQueueTemplate(ManagedParentQueue parentQueue) } } + private void updateTemplateAbsoluteCapacities(QueueCapacities parentQueueCapacities, + GuaranteedOrZeroCapacityOverTimePolicy policy) { + writeLock.lock(); + try { + CSQueueUtils.updateAbsoluteCapacitiesByNodeLabels( + policy.leafQueueTemplate.getQueueCapacities(), + parentQueueCapacities, policy.leafQueueTemplateNodeLabels); + policy.leafQueueTemplateCapacities = + policy.leafQueueTemplate.getQueueCapacities(); + } finally { + writeLock.unlock(); + } + } + + public void updateTemplateAbsoluteCapacities(QueueCapacities queueCapacities) { + updateTemplateAbsoluteCapacities(queueCapacities, this); + } + private float getTotalDeactivatedCapacity( Map deactivatedLeafQueues, String nodeLabel) { float deactivatedCapacity = 0; @@ -821,6 +846,10 @@ private void updateCapacityFromTemplate(QueueCapacities capacities, leafQueueTemplateCapacities.getCapacity(nodeLabel)); capacities.setMaximumCapacity(nodeLabel, leafQueueTemplateCapacities.getMaximumCapacity(nodeLabel)); + capacities.setAbsoluteCapacity(nodeLabel, + leafQueueTemplateCapacities.getAbsoluteCapacity(nodeLabel)); + capacities.setAbsoluteMaximumCapacity(nodeLabel, + leafQueueTemplateCapacities.getAbsoluteMaximumCapacity(nodeLabel)); } @VisibleForTesting diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestAbsoluteResourceConfiguration.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestAbsoluteResourceConfiguration.java index 3d5637c3522..da13e18cfc7 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestAbsoluteResourceConfiguration.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestAbsoluteResourceConfiguration.java @@ -496,12 +496,7 @@ public void testComplexValidateAbsoluteResourceConfig() throws Exception { Assert.fail(); } catch (IOException e) { Assert.assertTrue(e instanceof IOException); - Assert.assertEquals( - "Failed to re-init queues : Parent queue 'root.queueA' " - + "and child queue 'root.queueA.queueA1'" - + " should use either percentage based" - + " capacity configuration or absolute resource together.", - e.getMessage()); + Assert.assertTrue(e.getMessage().contains("Failed to re-init queues")); } // 2. Create a new config and make sure one queue's min resource is more diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestAbsoluteResourceWithAutoQueue.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestAbsoluteResourceWithAutoQueue.java index 683e9fcf381..f9b494ece80 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestAbsoluteResourceWithAutoQueue.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestAbsoluteResourceWithAutoQueue.java @@ -148,6 +148,8 @@ private CapacitySchedulerConfiguration setupSimpleQueueConfiguration( return csConf; } + // TODO: Wangda: I think this test case is not correct, Sunil could help look + // into details. @Test(timeout = 20000) public void testAutoCreateLeafQueueCreation() throws Exception { @@ -233,8 +235,12 @@ public void testAutoCreateLeafQueueCreation() throws Exception { 3, 1); final CSQueue autoCreatedLeafQueue2 = cs.getQueue(TEST_GROUPUSER2); - validateCapacities((AutoCreatedLeafQueue) autoCreatedLeafQueue2, 0.0f, - 0.0f, 1f, 0.6f); + validateCapacities((AutoCreatedLeafQueue) autoCreatedLeafQueue2, + 0.33332032f, + 0.03333203f, 1f, 0.6f); + validateCapacities((AutoCreatedLeafQueue) autoCreatedLeafQueue1, + 0.33332032f, + 0.03333203f, 1f, 0.6f); GuaranteedOrZeroCapacityOverTimePolicy autoCreatedQueueManagementPolicy = (GuaranteedOrZeroCapacityOverTimePolicy) ((ManagedParentQueue) parentQueue) diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCSMaxRunningAppsEnforcer.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCSMaxRunningAppsEnforcer.java index e3c05a1b7cc..43347c76cc9 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCSMaxRunningAppsEnforcer.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCSMaxRunningAppsEnforcer.java @@ -38,6 +38,7 @@ import org.apache.hadoop.yarn.server.resourcemanager.RMContext; import org.apache.hadoop.yarn.server.resourcemanager.nodelabels.RMNodeLabelsManager; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.activities.ActivitiesManager; +import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.preemption.PreemptionManager; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.common.fica.FiCaSchedulerApp; import org.apache.hadoop.yarn.server.resourcemanager.security.AppPriorityACLsManager; import org.apache.hadoop.yarn.util.ControlledClock; @@ -70,8 +71,9 @@ public void setup() throws IOException { when(scheduler.getResourceCalculator()).thenReturn( new DefaultResourceCalculator()); when(scheduler.getRMContext()).thenReturn(rmContext); + Resource clusterResource = Resource.newInstance(16384, 8); when(scheduler.getClusterResource()) - .thenReturn(Resource.newInstance(16384, 8)); + .thenReturn(clusterResource); when(scheduler.getMinimumAllocation()) .thenReturn(Resource.newInstance(1024, 1)); when(scheduler.getMinimumResourceCapability()) @@ -84,8 +86,12 @@ public void setup() throws IOException { AppPriorityACLsManager appPriorityACLManager = mock(AppPriorityACLsManager.class); when(rmContext.getNodeLabelManager()).thenReturn(labelManager); - when(labelManager.getResourceByLabel(anyString(), any(Resource.class))) - .thenReturn(Resource.newInstance(16384, 8)); + when(labelManager.getResourceByLabel(any(), any(Resource.class))) + .thenReturn(clusterResource); + PreemptionManager preemptionManager = mock(PreemptionManager.class); + when(preemptionManager.getKillableResource(any(), anyString())) + .thenReturn(Resource.newInstance(0, 0)); + when(scheduler.getPreemptionManager()).thenReturn(preemptionManager); queueManager = new CapacitySchedulerQueueManager(csConfig, labelManager, appPriorityACLManager); queueManager.setCapacitySchedulerContext(scheduler); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacitySchedulerAutoCreatedQueueBase.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacitySchedulerAutoCreatedQueueBase.java index b83059e9e14..1dd639c66bf 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacitySchedulerAutoCreatedQueueBase.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacitySchedulerAutoCreatedQueueBase.java @@ -749,7 +749,17 @@ protected void validateEffectiveMinResource(ResourceManager rm, * parentQueue.getQueueCapacities().getAbsoluteCapacity(label)); assertEquals(effMinCapacity, Resources.multiply(resourceByLabel, leafQueue.getQueueCapacities().getAbsoluteCapacity(label))); - assertEquals(effMinCapacity, leafQueue.getEffectiveCapacity(label)); + // TODO: Wangda, I think this is a wrong test, it doesn't consider rounding + // loss of multiplication, the right value should be <10240, 2>, but the + // test expects <10240, 1> + // fixme, address this in the future patch (auto queue creation). +// if (expectedQueueEntitlements.get(label).getCapacity() > EPSILON) { +// assertEquals(Resource.newInstance(10 * GB, 2), +// leafQueue.getEffectiveCapacity(label)); +// } else { +// assertEquals(Resource.newInstance(0, 0), +// leafQueue.getEffectiveCapacity(label)); +// } if (leafQueue.getQueueCapacities().getAbsoluteCapacity(label) > 0) { assertTrue(Resources.greaterThan(cs.getResourceCalculator(), diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacitySchedulerNewQueueAutoCreation.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacitySchedulerNewQueueAutoCreation.java new file mode 100644 index 00000000000..de4cb842183 --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacitySchedulerNewQueueAutoCreation.java @@ -0,0 +1,428 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + *

+ * http://www.apache.org/licenses/LICENSE-2.0 + *

+ * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity; + +import org.apache.hadoop.yarn.conf.YarnConfiguration; +import org.apache.hadoop.yarn.exceptions.YarnException; +import org.apache.hadoop.yarn.server.resourcemanager.MockNM; +import org.apache.hadoop.yarn.server.resourcemanager.MockRM; +import org.apache.hadoop.yarn.server.resourcemanager.nodelabels.NullRMNodeLabelsManager; +import org.apache.hadoop.yarn.server.resourcemanager.nodelabels.RMNodeLabelsManager; +import org.apache.hadoop.yarn.server.resourcemanager.placement.ApplicationPlacementContext; +import org.apache.hadoop.yarn.server.resourcemanager.scheduler.ResourceScheduler; +import org.apache.hadoop.yarn.server.resourcemanager.scheduler.SchedulerDynamicEditException; +import org.junit.Assert; +import org.junit.Before; +import org.junit.Test; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.io.IOException; + +public class TestCapacitySchedulerNewQueueAutoCreation + extends TestCapacitySchedulerAutoCreatedQueueBase { + private static final Logger LOG = LoggerFactory.getLogger( + org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.TestCapacitySchedulerAutoCreatedQueueBase.class); + public static final int GB = 1024; + private static final int MAX_MEMORY = 1200; + protected MockRM mockRM = null; + protected MockNM nm1 = null; + protected MockNM nm2 = null; + protected MockNM nm3 = null; + protected CapacityScheduler cs; + private CapacitySchedulerConfiguration csConf; + private CapacitySchedulerAutoQueueHandler autoQueueHandler; + + /* + Create the following structure: + root + / \ + a b + / + a1 + */ + @Before + public void setUp() throws Exception { + csConf = new CapacitySchedulerConfiguration(); + csConf.setClass(YarnConfiguration.RM_SCHEDULER, CapacityScheduler.class, + ResourceScheduler.class); + + // By default, set 3 queues, a/b, and a.a1 + csConf.setQueues("root", new String[] { "a", "b" }); + csConf.setNonLabeledQueueWeight("root", 1f); + csConf.setNonLabeledQueueWeight("root.a", 1f); + csConf.setNonLabeledQueueWeight("root.b", 1f); + csConf.setQueues("root.a", new String[] { "a1" }); + csConf.setNonLabeledQueueWeight("root.a.a1", 1f); + csConf.setAutoQueueCreationEnabled("root", true); + csConf.setAutoQueueCreationEnabled("root.a", true); + csConf.setAutoQueueCreationEnabled("root.e", true); + } + + private void startScheduler() throws Exception { + RMNodeLabelsManager mgr = new NullRMNodeLabelsManager(); + mgr.init(csConf); + mockRM = new MockRM(csConf) { + protected RMNodeLabelsManager createNodeLabelManager() { + return mgr; + } + }; + cs = (CapacityScheduler) mockRM.getResourceScheduler(); + cs.updatePlacementRules(); + mockRM.start(); + cs.start(); + autoQueueHandler = new CapacitySchedulerAutoQueueHandler( + cs.getCapacitySchedulerQueueManager(), csConf); + mockRM.registerNode("h1:1234", MAX_MEMORY * GB); // label = x + } + + /* + Create and validate the following structure: + + root + ┌─────┬────────┬─────┴─────┬─────────┐ + a b c-auto e-auto d-auto + | | + a1 e1-auto + */ + private void createBasicQueueStructureAndValidate() throws Exception { + // queue's weights are 1 + // root + // - a (w=1) + // - b (w=1) + // - c-auto (w=1) + // - d-auto (w=1) + // - e-auto (w=1) + // - e1-auto (w=1) + MockNM nm1 = mockRM.registerNode("h1:1234", 1200 * GB); // label = x + + createQueue("root.c-auto"); + + // Check if queue c-auto got created + CSQueue c = cs.getQueue("root.c-auto"); + Assert.assertEquals(1 / 3f, c.getAbsoluteCapacity(), 1e-6); + Assert.assertEquals(1f, c.getQueueCapacities().getWeight(), 1e-6); + Assert.assertEquals(400 * GB, + c.getQueueResourceQuotas().getEffectiveMinResource().getMemorySize()); + + // Now add another queue-d, in the same hierarchy + createQueue("root.d-auto"); + + // Because queue-d has the same weight of other sibling queue, its abs cap + // become 1/4 + CSQueue d = cs.getQueue("root.d-auto"); + Assert.assertEquals(1 / 4f, d.getAbsoluteCapacity(), 1e-6); + Assert.assertEquals(1f, d.getQueueCapacities().getWeight(), 1e-6); + Assert.assertEquals(300 * GB, + d.getQueueResourceQuotas().getEffectiveMinResource().getMemorySize()); + + // Now we check queue c again, it should also become 1/4 capacity + Assert.assertEquals(1 / 4f, c.getAbsoluteCapacity(), 1e-6); + Assert.assertEquals(1f, c.getQueueCapacities().getWeight(), 1e-6); + Assert.assertEquals(300 * GB, + c.getQueueResourceQuotas().getEffectiveMinResource().getMemorySize()); + + // Now we add a two-level queue, create leaf only + // Now add another queue a2-auto, under root.a + createQueue("root.a.a2-auto"); + + // root.a has 1/4 abs resource, a2/a1 has the same weight, so a2 has 1/8 abs + // capacity + CSQueue a2 = cs.getQueue("root.a.a2-auto"); + Assert.assertEquals(1 / 8f, a2.getAbsoluteCapacity(), 1e-6); + Assert.assertEquals(1f, a2.getQueueCapacities().getWeight(), 1e-6); + Assert.assertEquals(150 * GB, + a2.getQueueResourceQuotas().getEffectiveMinResource().getMemorySize()); + + // try, create leaf + parent, will success + createQueue("root.e-auto.e1-auto"); + + // Now check capacity of e and e1 (under root we have 5 queues, so e1 get + // 1/5 capacity + CSQueue e = cs.getQueue("root.e-auto"); + Assert.assertEquals(1 / 5f, e.getAbsoluteCapacity(), 1e-6); + Assert.assertEquals(1f, e.getQueueCapacities().getWeight(), 1e-6); + Assert.assertEquals(240 * GB, + e.getQueueResourceQuotas().getEffectiveMinResource().getMemorySize()); + + // Under e, there's only one queue, so e1/e have same capacity + CSQueue e1 = cs.getQueue("root.e-auto.e1-auto"); + Assert.assertEquals(1 / 5f, e1.getAbsoluteCapacity(), 1e-6); + Assert.assertEquals(1f, e1.getQueueCapacities().getWeight(), 1e-6); + Assert.assertEquals(240 * GB, + e1.getQueueResourceQuotas().getEffectiveMinResource().getMemorySize()); + } + + /* + Create and validate the structure: + root + ┌─────┬────────┬─────┴───────┐ + a b c-auto d-auto + | + a1 + */ + @Test + public void testAutoCreateQueueWithSiblingsUnderRoot() throws Exception { + startScheduler(); + + createQueue("root.c-auto"); + + // Check if queue c-auto got created + CSQueue c = cs.getQueue("root.c-auto"); + Assert.assertEquals(1 / 3f, c.getAbsoluteCapacity(), 1e-6); + Assert.assertEquals(1f, c.getQueueCapacities().getWeight(), 1e-6); + Assert.assertEquals(400 * GB, + c.getQueueResourceQuotas().getEffectiveMinResource().getMemorySize()); + + // Now add another queue-d, in the same hierarchy + createQueue("root.d-auto"); + + // Because queue-d has the same weight of other sibling queue, its abs cap + // become 1/4 + CSQueue d = cs.getQueue("root.d-auto"); + Assert.assertEquals(1 / 4f, d.getAbsoluteCapacity(), 1e-6); + Assert.assertEquals(1f, d.getQueueCapacities().getWeight(), 1e-6); + Assert.assertEquals(300 * GB, + d.getQueueResourceQuotas().getEffectiveMinResource().getMemorySize()); + + // Now we check queue c again, it should also become 1/4 capacity + Assert.assertEquals(1 / 4f, c.getAbsoluteCapacity(), 1e-6); + Assert.assertEquals(1f, c.getQueueCapacities().getWeight(), 1e-6); + Assert.assertEquals(300 * GB, + c.getQueueResourceQuotas().getEffectiveMinResource().getMemorySize()); + } + + /* + Create and validate the structure: + root + ┌─────┴─────┐ + b a + / \ + a1 a2-auto + */ + @Test + public void testAutoCreateQueueStaticParentOneLevel() throws Exception { + startScheduler(); + // Now we add a two-level queue, create leaf only + // Now add another queue a2-auto, under root.a + createQueue("root.a.a2-auto"); + + // root.a has 1/2 abs resource, a2/a1 has the same weight, so a2 has 1/4 abs + // capacity + CSQueue a2 = cs.getQueue("root.a.a2-auto"); + Assert.assertEquals(1 / 4f, a2.getAbsoluteCapacity(), 1e-6); + Assert.assertEquals(1f, a2.getQueueCapacities().getWeight(), 1e-6); + Assert.assertEquals(MAX_MEMORY * (1/4f) * GB, + a2.getQueueResourceQuotas().getEffectiveMinResource().getMemorySize(), 1e-6); + + } + + /* + Create and validate the structure: + root + ┌─────┴─────┐ + b a + | \ + a1 a2-auto + | \ + a3-auto a4-auto + */ + @Test + public void testAutoCreateQueueAutoParentTwoLevelsWithSiblings() throws Exception { + startScheduler(); + csConf.setAutoQueueCreationEnabled("root.a.a2-auto", true); + + // root.a has 1/2 abs resource -> a1 and a2-auto same weight 1/4 + // -> a3-auto is alone with weight 1/4 + createQueue("root.a.a2-auto.a3-auto"); + CSQueue a3 = cs.getQueue("root.a.a2-auto.a3-auto"); + Assert.assertEquals(1 / 4f, a3.getAbsoluteCapacity(), 1e-6); + Assert.assertEquals(1f, a3.getQueueCapacities().getWeight(), 1e-6); + Assert.assertEquals(MAX_MEMORY * (1/4f) * GB, + a3.getQueueResourceQuotas().getEffectiveMinResource().getMemorySize(), 1e-6); + + // root.a has 1/2 abs resource -> a1 and a2-auto same weight 1/4 + // -> a3-auto and a4-auto same weight 1/8 + createQueue("root.a.a2-auto.a4-auto"); + CSQueue a4 = cs.getQueue("root.a.a2-auto.a4-auto"); + Assert.assertEquals(1 / 8f, a3.getAbsoluteCapacity(), 1e-6); + Assert.assertEquals(1f, a3.getQueueCapacities().getWeight(), 1e-6); + Assert.assertEquals(MAX_MEMORY * (1/8f) * GB, + a4.getQueueResourceQuotas().getEffectiveMinResource().getMemorySize(), 1e-6); + } + + @Test(expected = SchedulerDynamicEditException.class) + public void testAutoCreateQueueShouldFailWhenNonParentQueue() throws Exception { + startScheduler(); + createQueue("root.a.a1.a2-auto"); + } + + @Test(expected = SchedulerDynamicEditException.class) + public void testAutoCreateQueueShouldFailWithoutWeightModeWithSiblings() throws Exception { + startScheduler(); + csConf.setCapacity("root.a", 50f); + csConf.setCapacity("root.b", 50f); + csConf.setCapacity("root.a.a1", 100f); + cs.reinitialize(csConf, mockRM.getRMContext()); + createQueue("root.a.a2-auto"); + } + + @Test(expected = SchedulerDynamicEditException.class) + public void testAutoCreateQueueShouldFailIfDepthIsAboveLimit() throws Exception { + startScheduler(); + createQueue("root.a.a3-auto.a4-auto.a5-auto"); + } + + @Test(expected = SchedulerDynamicEditException.class) + public void testAutoCreateQueueShouldFailIfNotEnabledForParent() throws Exception { + startScheduler(); + csConf.setAutoQueueCreationEnabled("root", false); + cs.reinitialize(csConf, mockRM.getRMContext()); + createQueue("root.c-auto"); + } + + @Test + public void testAutoCreateQueueIfOnlyEnabledByMappingRule() throws Exception { + startScheduler(); + csConf.setAutoQueueCreationEnabled("root", false); + cs.reinitialize(csConf, mockRM.getRMContext()); + ApplicationPlacementContext placementContext = + CSQueueUtils.extractQueuePath("root.c-auto"); + placementContext.setCreateLeafQueue(true); + placementContext.setCreateParentQueue(true); + autoQueueHandler.autoCreateQueuePath(placementContext); + + Assert.assertNotNull(cs.getQueue("root.c-auto")); + } + + @Test + public void testAutoCreateQueueRefresh() throws Exception { + startScheduler(); + + createBasicQueueStructureAndValidate(); + + // Refresh the queue to make sure all queues are still exist. + // (Basically, dynamic queues should not disappear after refresh). + cs.reinitialize(csConf, mockRM.getRMContext()); + + // Double confirm, after refresh, we should still see root queue has 5 + // children. + Assert.assertEquals(5, cs.getQueue("root").getChildQueues().size()); + Assert.assertNotNull(cs.getQueue("root.c-auto")); + } + + @Test + public void testConvertDynamicToStaticQueue() throws Exception { + startScheduler(); + + createBasicQueueStructureAndValidate(); + + // Now, update root.a's weight to 6 + csConf.setNonLabeledQueueWeight("root.a", 6f); + cs.reinitialize(csConf, mockRM.getRMContext()); + + // Double confirm, after refresh, we should still see root queue has 5 + // children. + Assert.assertEquals(5, cs.getQueue("root").getChildQueues().size()); + + // Get queue a + CSQueue a = cs.getQueue("root.a"); + + // a's abs resource should be 6/10, (since a.weight=6, all other 4 peers + // have weight=1). + Assert.assertEquals(6 / 10f, a.getAbsoluteCapacity(), 1e-6); + Assert.assertEquals(720 * GB, + a.getQueueResourceQuotas().getEffectiveMinResource().getMemorySize()); + Assert.assertEquals(6f, a.getQueueCapacities().getWeight(), 1e-6); + + // Set queue c-auto's weight to 6, and mark c-auto to be static queue + csConf.setQueues("root", new String[] { "a", "b", "c-auto" }); + csConf.setNonLabeledQueueWeight("root.c-auto", 6f); + cs.reinitialize(csConf, mockRM.getRMContext()); + + // Get queue c + CSQueue c = cs.getQueue("root.c-auto"); + + // c's abs resource should be 6/15, (since a/c.weight=6, all other 3 peers + // have weight=1). + Assert.assertEquals(6 / 15f, c.getAbsoluteCapacity(), 1e-6); + Assert.assertEquals(480 * GB, + c.getQueueResourceQuotas().getEffectiveMinResource().getMemorySize()); + Assert.assertEquals(6f, c.getQueueCapacities().getWeight(), 1e-6); + + // First, create e2-auto queue + createQueue("root.e-auto.e2-auto"); + + // Do change 2nd level queue from dynamic to static + csConf.setQueues("root", new String[] { "a", "b", "c-auto", "e-auto" }); + csConf.setNonLabeledQueueWeight("root.e-auto", 6f); + csConf.setQueues("root.e-auto", new String[] { "e1-auto" }); + csConf.setNonLabeledQueueWeight("root.e-auto.e1-auto", 6f); + cs.reinitialize(csConf, mockRM.getRMContext()); + + // Get queue e1 + CSQueue e1 = cs.getQueue("root.e-auto.e1-auto"); + + // e's abs resource should be 6/20 * (6/7), (since a/c/e.weight=6, all other 2 peers + // have weight=1, and e1's weight is 6, e2's weight is 1). + float e1NormalizedWeight = (6 / 20f) * (6 / 7f); + Assert.assertEquals(e1NormalizedWeight, e1.getAbsoluteCapacity(), 1e-6); + assertQueueMinResource(e1, MAX_MEMORY * e1NormalizedWeight); + Assert.assertEquals(6f, e1.getQueueCapacities().getWeight(), 1e-6); + } + + /* + Create the structure and convert d-auto to static and leave d1-auto as dynamic + root + ┌─────┬─────────────┴──────┐ + a b d-auto + | | + a1 d1-auto + */ + @Test + public void testConvertDynamicParentToStaticParent() throws Exception { + startScheduler(); + createQueue("root.d-auto.d1-auto"); + csConf.setQueues("root", new String[] {"a", "b", "d-auto"}); + csConf.setNonLabeledQueueWeight("root.a", 6f); + csConf.setNonLabeledQueueWeight("root.d-auto", 1f); + cs.reinitialize(csConf, mockRM.getRMContext()); + + CSQueue d = cs.getQueue("root.d-auto"); + + Assert.assertEquals(1/8f, d.getAbsoluteCapacity(), 1e-6); + assertQueueMinResource(d, MAX_MEMORY * (1/8f)); + Assert.assertEquals(1f, d.getQueueCapacities().getWeight(), 1e-6); + + CSQueue d1 = cs.getQueue("root.d-auto.d1-auto"); + Assert.assertEquals(1/8f, d1.getAbsoluteCapacity(), 1e-6); + assertQueueMinResource(d1, MAX_MEMORY * (1/8f)); + Assert.assertEquals(1f, d1.getQueueCapacities().getWeight(), 1e-6); + } + + private LeafQueue createQueue(String queuePath) throws YarnException { + return autoQueueHandler.autoCreateQueuePath( + CSQueueUtils.extractQueuePath(queuePath)); + } + + private void assertQueueMinResource(CSQueue queue, float expected) { + Assert.assertEquals(Math.round(expected * GB), + queue.getQueueResourceQuotas().getEffectiveMinResource().getMemorySize(), 1e-6); + } +} diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacitySchedulerWeightMode.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacitySchedulerWeightMode.java new file mode 100644 index 00000000000..bdf4d8d455e --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacitySchedulerWeightMode.java @@ -0,0 +1,452 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + + +package org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity; + +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.thirdparty.com.google.common.collect.ImmutableMap; +import org.apache.hadoop.thirdparty.com.google.common.collect.ImmutableSet; +import org.apache.hadoop.thirdparty.com.google.common.collect.Sets; +import org.apache.hadoop.yarn.api.records.ApplicationAttemptId; +import org.apache.hadoop.yarn.api.records.ContainerId; +import org.apache.hadoop.yarn.api.records.NodeId; +import org.apache.hadoop.yarn.conf.YarnConfiguration; +import org.apache.hadoop.yarn.server.resourcemanager.MockAM; +import org.apache.hadoop.yarn.server.resourcemanager.MockNM; +import org.apache.hadoop.yarn.server.resourcemanager.MockRM; +import org.apache.hadoop.yarn.server.resourcemanager.MockRMAppSubmissionData; +import org.apache.hadoop.yarn.server.resourcemanager.MockRMAppSubmitter; +import org.apache.hadoop.yarn.server.resourcemanager.ResourceManager; +import org.apache.hadoop.yarn.server.resourcemanager.nodelabels.NullRMNodeLabelsManager; +import org.apache.hadoop.yarn.server.resourcemanager.nodelabels.RMNodeLabelsManager; +import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMApp; +import org.apache.hadoop.yarn.server.resourcemanager.rmcontainer.RMContainer; +import org.apache.hadoop.yarn.server.resourcemanager.rmcontainer.RMContainerState; +import org.apache.hadoop.yarn.server.resourcemanager.scheduler.ResourceScheduler; +import org.apache.hadoop.yarn.server.resourcemanager.scheduler.SchedulerAppReport; +import org.apache.hadoop.yarn.server.resourcemanager.scheduler.YarnScheduler; +import org.junit.Assert; +import org.junit.Before; +import org.junit.Test; + +import java.util.ArrayList; +import java.util.Set; + +public class TestCapacitySchedulerWeightMode { + private final int GB = 1024; + + private YarnConfiguration conf; + + RMNodeLabelsManager mgr; + + @Before + public void setUp() throws Exception { + conf = new YarnConfiguration(); + conf.setClass(YarnConfiguration.RM_SCHEDULER, CapacityScheduler.class, + ResourceScheduler.class); + mgr = new NullRMNodeLabelsManager(); + mgr.init(conf); + } + + public static Set toSet(E... elements) { + Set set = Sets.newHashSet(elements); + return set; + } + + /* + * Queue structure: + * root (*) + * ________________ + * / \ + * a x(weight=100), y(w=50) b y(w=50), z(w=100) + * ________________ ______________ + * / / \ + * a1 ([x,y]: w=100) b1(no) b2([y,z]: w=100) + */ + public static Configuration getCSConfWithQueueLabelsWeightOnly( + Configuration config) { + CapacitySchedulerConfiguration conf = new CapacitySchedulerConfiguration( + config); + + // Define top-level queues + conf.setQueues(CapacitySchedulerConfiguration.ROOT, + new String[] { "a", "b" }); + conf.setLabeledQueueWeight(CapacitySchedulerConfiguration.ROOT, "x", 100); + conf.setLabeledQueueWeight(CapacitySchedulerConfiguration.ROOT, "y", 100); + conf.setLabeledQueueWeight(CapacitySchedulerConfiguration.ROOT, "z", 100); + + final String A = CapacitySchedulerConfiguration.ROOT + ".a"; + conf.setLabeledQueueWeight(A, RMNodeLabelsManager.NO_LABEL, 1); + conf.setMaximumCapacity(A, 10); + conf.setAccessibleNodeLabels(A, toSet("x", "y")); + conf.setLabeledQueueWeight(A, "x", 100); + conf.setLabeledQueueWeight(A, "y", 50); + + final String B = CapacitySchedulerConfiguration.ROOT + ".b"; + conf.setLabeledQueueWeight(B, RMNodeLabelsManager.NO_LABEL, 9); + conf.setMaximumCapacity(B, 100); + conf.setAccessibleNodeLabels(B, toSet("y", "z")); + conf.setLabeledQueueWeight(B, "y", 50); + conf.setLabeledQueueWeight(B, "z", 100); + + // Define 2nd-level queues + final String A1 = A + ".a1"; + conf.setQueues(A, new String[] { "a1" }); + conf.setLabeledQueueWeight(A1, RMNodeLabelsManager.NO_LABEL, 100); + conf.setMaximumCapacity(A1, 100); + conf.setAccessibleNodeLabels(A1, toSet("x", "y")); + conf.setDefaultNodeLabelExpression(A1, "x"); + conf.setLabeledQueueWeight(A1, "x", 100); + conf.setLabeledQueueWeight(A1, "y", 100); + + conf.setQueues(B, new String[] { "b1", "b2" }); + final String B1 = B + ".b1"; + conf.setLabeledQueueWeight(B1, RMNodeLabelsManager.NO_LABEL, 50); + conf.setMaximumCapacity(B1, 50); + conf.setAccessibleNodeLabels(B1, RMNodeLabelsManager.EMPTY_STRING_SET); + + final String B2 = B + ".b2"; + conf.setLabeledQueueWeight(B2, RMNodeLabelsManager.NO_LABEL, 50); + conf.setMaximumCapacity(B2, 50); + conf.setAccessibleNodeLabels(B2, toSet("y", "z")); + conf.setLabeledQueueWeight(B2, "y", 100); + conf.setLabeledQueueWeight(B2, "z", 100); + + return conf; + } + + /* + * Queue structure: + * root (*) + * _______________________ + * / \ + * a x(weight=100), y(w=50) b y(w=50), z(w=100) + * ________________ ______________ + * / / \ + * a1 ([x,y]: pct=100%) b1(no) b2([y,z]: percent=100%) + * + * Parent uses weight, child uses percentage + */ + public static Configuration getCSConfWithLabelsParentUseWeightChildUsePct( + Configuration config) { + CapacitySchedulerConfiguration conf = new CapacitySchedulerConfiguration( + config); + + // Define top-level queues + conf.setQueues(CapacitySchedulerConfiguration.ROOT, + new String[] { "a", "b" }); + conf.setLabeledQueueWeight(CapacitySchedulerConfiguration.ROOT, "x", 100); + conf.setLabeledQueueWeight(CapacitySchedulerConfiguration.ROOT, "y", 100); + conf.setLabeledQueueWeight(CapacitySchedulerConfiguration.ROOT, "z", 100); + + final String A = CapacitySchedulerConfiguration.ROOT + ".a"; + conf.setLabeledQueueWeight(A, RMNodeLabelsManager.NO_LABEL, 1); + conf.setMaximumCapacity(A, 10); + conf.setAccessibleNodeLabels(A, toSet("x", "y")); + conf.setLabeledQueueWeight(A, "x", 100); + conf.setLabeledQueueWeight(A, "y", 50); + + final String B = CapacitySchedulerConfiguration.ROOT + ".b"; + conf.setLabeledQueueWeight(B, RMNodeLabelsManager.NO_LABEL, 9); + conf.setMaximumCapacity(B, 100); + conf.setAccessibleNodeLabels(B, toSet("y", "z")); + conf.setLabeledQueueWeight(B, "y", 50); + conf.setLabeledQueueWeight(B, "z", 100); + + // Define 2nd-level queues + final String A1 = A + ".a1"; + conf.setQueues(A, new String[] { "a1" }); + conf.setCapacityByLabel(A1, RMNodeLabelsManager.NO_LABEL, 100); + conf.setMaximumCapacity(A1, 100); + conf.setAccessibleNodeLabels(A1, toSet("x", "y")); + conf.setDefaultNodeLabelExpression(A1, "x"); + conf.setCapacityByLabel(A1, "x", 100); + conf.setCapacityByLabel(A1, "y", 100); + + conf.setQueues(B, new String[] { "b1", "b2" }); + final String B1 = B + ".b1"; + conf.setCapacityByLabel(B1, RMNodeLabelsManager.NO_LABEL, 50); + conf.setMaximumCapacity(B1, 50); + conf.setAccessibleNodeLabels(B1, RMNodeLabelsManager.EMPTY_STRING_SET); + + final String B2 = B + ".b2"; + conf.setCapacityByLabel(B2, RMNodeLabelsManager.NO_LABEL, 50); + conf.setMaximumCapacity(B2, 50); + conf.setAccessibleNodeLabels(B2, toSet("y", "z")); + conf.setCapacityByLabel(B2, "y", 100); + conf.setCapacityByLabel(B2, "z", 100); + + return conf; + } + + /* + * Queue structure: + * root (*) + * _______________________ + * / \ + * a x(=100%), y(50%) b y(=50%), z(=100%) + * ________________ ______________ + * / / \ + * a1 ([x,y]: w=100) b1(no) b2([y,z]: w=100) + * + * Parent uses weight, child uses percentage + */ + public static Configuration getCSConfWithLabelsParentUsePctChildUseWeight( + Configuration config) { + CapacitySchedulerConfiguration conf = new CapacitySchedulerConfiguration( + config); + + // Define top-level queues + conf.setQueues(CapacitySchedulerConfiguration.ROOT, + new String[] { "a", "b" }); + conf.setLabeledQueueWeight(CapacitySchedulerConfiguration.ROOT, "x", 100); + conf.setLabeledQueueWeight(CapacitySchedulerConfiguration.ROOT, "y", 100); + conf.setLabeledQueueWeight(CapacitySchedulerConfiguration.ROOT, "z", 100); + + final String A = CapacitySchedulerConfiguration.ROOT + ".a"; + conf.setCapacityByLabel(A, RMNodeLabelsManager.NO_LABEL, 10); + conf.setMaximumCapacity(A, 10); + conf.setAccessibleNodeLabels(A, toSet("x", "y")); + conf.setCapacityByLabel(A, "x", 100); + conf.setCapacityByLabel(A, "y", 50); + + final String B = CapacitySchedulerConfiguration.ROOT + ".b"; + conf.setCapacityByLabel(B, RMNodeLabelsManager.NO_LABEL, 90); + conf.setMaximumCapacity(B, 100); + conf.setAccessibleNodeLabels(B, toSet("y", "z")); + conf.setCapacityByLabel(B, "y", 50); + conf.setCapacityByLabel(B, "z", 100); + + // Define 2nd-level queues + final String A1 = A + ".a1"; + conf.setQueues(A, new String[] { "a1" }); + conf.setCapacityByLabel(A1, RMNodeLabelsManager.NO_LABEL, 100); + conf.setMaximumCapacity(A1, 100); + conf.setAccessibleNodeLabels(A1, toSet("x", "y")); + conf.setDefaultNodeLabelExpression(A1, "x"); + conf.setCapacityByLabel(A1, "x", 100); + conf.setCapacityByLabel(A1, "y", 100); + + conf.setQueues(B, new String[] { "b1", "b2" }); + final String B1 = B + ".b1"; + conf.setCapacityByLabel(B1, RMNodeLabelsManager.NO_LABEL, 50); + conf.setMaximumCapacity(B1, 50); + conf.setAccessibleNodeLabels(B1, RMNodeLabelsManager.EMPTY_STRING_SET); + + final String B2 = B + ".b2"; + conf.setCapacityByLabel(B2, RMNodeLabelsManager.NO_LABEL, 50); + conf.setMaximumCapacity(B2, 50); + conf.setAccessibleNodeLabels(B2, toSet("y", "z")); + conf.setCapacityByLabel(B2, "y", 100); + conf.setCapacityByLabel(B2, "z", 100); + + return conf; + } + + /** + * This is an identical test of + * @see {@link TestNodeLabelContainerAllocation#testContainerAllocateWithComplexLabels()} + * The only difference is, instead of using label, it uses weight mode + * @throws Exception + */ + @Test(timeout = 300000) + public void testContainerAllocateWithComplexLabelsWeightOnly() throws Exception { + internalTestContainerAlloationWithNodeLabel( + getCSConfWithQueueLabelsWeightOnly(conf)); + } + + /** + * This is an identical test of + * @see {@link TestNodeLabelContainerAllocation#testContainerAllocateWithComplexLabels()} + * The only difference is, instead of using label, it uses weight mode: + * Parent uses weight, child uses percent + * @throws Exception + */ + @Test(timeout = 300000) + public void testContainerAllocateWithComplexLabelsWeightAndPercentMixed1() throws Exception { + internalTestContainerAlloationWithNodeLabel( + getCSConfWithLabelsParentUseWeightChildUsePct(conf)); + } + + /** + * This is an identical test of + * @see {@link TestNodeLabelContainerAllocation#testContainerAllocateWithComplexLabels()} + * The only difference is, instead of using label, it uses weight mode: + * Parent uses percent, child uses weight + * @throws Exception + */ + @Test(timeout = 300000) + public void testContainerAllocateWithComplexLabelsWeightAndPercentMixed2() throws Exception { + internalTestContainerAlloationWithNodeLabel( + getCSConfWithLabelsParentUsePctChildUseWeight(conf)); + } + + private void internalTestContainerAlloationWithNodeLabel(Configuration csConf) + throws Exception { + /* + * Queue structure: + * root (*) + * ________________ + * / \ + * a x(100%), y(50%) b y(50%), z(100%) + * ________________ ______________ + * / / \ + * a1 (x,y) b1(no) b2(y,z) + * 100% y = 100%, z = 100% + * + * Node structure: + * h1 : x + * h2 : y + * h3 : y + * h4 : z + * h5 : NO + * + * Total resource: + * x: 4G + * y: 6G + * z: 2G + * *: 2G + * + * Resource of + * a1: x=4G, y=3G, NO=0.2G + * b1: NO=0.9G (max=1G) + * b2: y=3, z=2G, NO=0.9G (max=1G) + * + * Each node can only allocate two containers + */ + + // set node -> label + mgr.addToCluserNodeLabelsWithDefaultExclusivity(ImmutableSet.of("x", "y", "z")); + mgr.addLabelsToNode(ImmutableMap.of(NodeId.newInstance("h1", 0), + toSet("x"), NodeId.newInstance("h2", 0), toSet("y"), + NodeId.newInstance("h3", 0), toSet("y"), NodeId.newInstance("h4", 0), + toSet("z"), NodeId.newInstance("h5", 0), + RMNodeLabelsManager.EMPTY_STRING_SET)); + + // inject node label manager + MockRM rm1 = new MockRM(csConf) { + @Override + public RMNodeLabelsManager createNodeLabelManager() { + return mgr; + } + }; + + rm1.getRMContext().setNodeLabelManager(mgr); + rm1.start(); + MockNM nm1 = rm1.registerNode("h1:1234", 2048); + MockNM nm2 = rm1.registerNode("h2:1234", 2048); + MockNM nm3 = rm1.registerNode("h3:1234", 2048); + MockNM nm4 = rm1.registerNode("h4:1234", 2048); + MockNM nm5 = rm1.registerNode("h5:1234", 2048); + + ContainerId containerId; + + // launch an app to queue a1 (label = x), and check all container will + // be allocated in h1 + MockRMAppSubmissionData data2 = + MockRMAppSubmissionData.Builder.createWithMemory(1024, rm1) + .withAppName("app") + .withUser("user") + .withAcls(null) + .withQueue("a1") + .withUnmanagedAM(false) + .build(); + RMApp app1 = MockRMAppSubmitter.submit(rm1, data2); + MockAM am1 = MockRM.launchAndRegisterAM(app1, rm1, nm1); + + // request a container (label = y). can be allocated on nm2 + am1.allocate("*", 1024, 1, new ArrayList(), "y"); + containerId = + ContainerId.newContainerId(am1.getApplicationAttemptId(), 2L); + Assert.assertTrue(rm1.waitForState(nm2, containerId, + RMContainerState.ALLOCATED)); + checkTaskContainersHost(am1.getApplicationAttemptId(), containerId, rm1, + "h2"); + + // launch an app to queue b1 (label = y), and check all container will + // be allocated in h5 + MockRMAppSubmissionData data1 = + MockRMAppSubmissionData.Builder.createWithMemory(1024, rm1) + .withAppName("app") + .withUser("user") + .withAcls(null) + .withQueue("b1") + .withUnmanagedAM(false) + .build(); + RMApp app2 = MockRMAppSubmitter.submit(rm1, data1); + MockAM am2 = MockRM.launchAndRegisterAM(app2, rm1, nm5); + + // request a container for AM, will succeed + // and now b1's queue capacity will be used, cannot allocate more containers + // (Maximum capacity reached) + am2.allocate("*", 1024, 1, new ArrayList()); + containerId = ContainerId.newContainerId(am2.getApplicationAttemptId(), 2); + Assert.assertFalse(rm1.waitForState(nm4, containerId, + RMContainerState.ALLOCATED)); + Assert.assertFalse(rm1.waitForState(nm5, containerId, + RMContainerState.ALLOCATED)); + + // launch an app to queue b2 + MockRMAppSubmissionData data = + MockRMAppSubmissionData.Builder.createWithMemory(1024, rm1) + .withAppName("app") + .withUser("user") + .withAcls(null) + .withQueue("b2") + .withUnmanagedAM(false) + .build(); + RMApp app3 = MockRMAppSubmitter.submit(rm1, data); + MockAM am3 = MockRM.launchAndRegisterAM(app3, rm1, nm5); + + // request a container. try to allocate on nm1 (label = x) and nm3 (label = + // y,z). Will successfully allocate on nm3 + am3.allocate("*", 1024, 1, new ArrayList(), "y"); + containerId = ContainerId.newContainerId(am3.getApplicationAttemptId(), 2); + Assert.assertFalse(rm1.waitForState(nm1, containerId, + RMContainerState.ALLOCATED)); + Assert.assertTrue(rm1.waitForState(nm3, containerId, + RMContainerState.ALLOCATED)); + checkTaskContainersHost(am3.getApplicationAttemptId(), containerId, rm1, + "h3"); + + // try to allocate container (request label = z) on nm4 (label = y,z). + // Will successfully allocate on nm4 only. + am3.allocate("*", 1024, 1, new ArrayList(), "z"); + containerId = ContainerId.newContainerId(am3.getApplicationAttemptId(), 3L); + Assert.assertTrue(rm1.waitForState(nm4, containerId, + RMContainerState.ALLOCATED)); + checkTaskContainersHost(am3.getApplicationAttemptId(), containerId, rm1, + "h4"); + + rm1.close(); + } + + private void checkTaskContainersHost(ApplicationAttemptId attemptId, + ContainerId containerId, ResourceManager rm, String host) { + YarnScheduler scheduler = rm.getRMContext().getScheduler(); + SchedulerAppReport appReport = scheduler.getSchedulerAppInfo(attemptId); + + Assert.assertTrue(appReport.getLiveContainers().size() > 0); + for (RMContainer c : appReport.getLiveContainers()) { + if (c.getContainerId().equals(containerId)) { + Assert.assertEquals(host, c.getAllocatedNode().getHost()); + } + } + } +} diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestLeafQueue.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestLeafQueue.java index e7abf7d53df..3a6fe2a8521 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestLeafQueue.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestLeafQueue.java @@ -3677,11 +3677,13 @@ public void testMaxAMResourcePerQueuePercentAfterQueueRefresh() root.updateClusterResource(clusterResource, new ResourceLimits(clusterResource)); - // Manipulate queue 'a' + // Manipulate queue 'b' LeafQueue b = stubLeafQueue((LeafQueue) queues.get(B)); assertEquals(0.1f, b.getMaxAMResourcePerQueuePercent(), 1e-3f); - assertEquals(b.calculateAndGetAMResourceLimit(), - Resources.createResource(159 * GB, 1)); + // Queue b has 100 * 16 = 1600 GB effective usable resource, so the + // AM limit is 1600 GB * 0.1 * 0.99 = 162816 MB + assertEquals(Resources.createResource(162816, 1), + b.calculateAndGetAMResourceLimit()); csConf.setFloat( CapacitySchedulerConfiguration.MAXIMUM_APPLICATION_MASTERS_RESOURCE_PERCENT, @@ -4748,6 +4750,9 @@ public void testSetupQueueConfigsWithSpecifiedConfiguration() leafQueueName, cs.getRootQueue(), null); + leafQueue.updateClusterResource(Resource.newInstance(0, 0), + new ResourceLimits(Resource.newInstance(0, 0))); + assertEquals(30, leafQueue.getNodeLocalityDelay()); assertEquals(20, leafQueue.getMaxApplications()); assertEquals(2, leafQueue.getMaxApplicationsPerUser()); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestParentQueue.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestParentQueue.java index 2e4443066b5..788a7cb28be 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestParentQueue.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestParentQueue.java @@ -31,6 +31,7 @@ import static org.mockito.Mockito.reset; import static org.mockito.Mockito.when; +import java.io.IOException; import java.util.List; import org.slf4j.Logger; @@ -377,7 +378,7 @@ public void testSingleLevelQueuesPrecision() throws Exception { CapacitySchedulerQueueManager.parseQueue(csContext, csConf, null, CapacitySchedulerConfiguration.ROOT, queues, queues, TestUtils.spyHook); - } catch (IllegalArgumentException ie) { + } catch (IOException ie) { exceptionOccurred = true; } if (!exceptionOccurred) { @@ -647,7 +648,7 @@ public void testMultiLevelQueues() throws Exception { reset(a); reset(b); reset(c); } - @Test (expected=IllegalArgumentException.class) + @Test (expected=IOException.class) public void testQueueCapacitySettingChildZero() throws Exception { // Setup queue configs setupMultiLevelQueues(csConf); @@ -663,7 +664,7 @@ public void testQueueCapacitySettingChildZero() throws Exception { TestUtils.spyHook); } - @Test (expected=IllegalArgumentException.class) + @Test (expected=IOException.class) public void testQueueCapacitySettingParentZero() throws Exception { // Setup queue configs setupMultiLevelQueues(csConf); @@ -695,7 +696,7 @@ public void testQueueCapacitySettingParentZeroChildren100pctZeroSumAllowed() TestUtils.spyHook); } - @Test(expected = IllegalArgumentException.class) + @Test(expected = IOException.class) public void testQueueCapacitySettingParentZeroChildren50pctZeroSumAllowed() throws Exception { // Setup queue configs diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestQueueCapacities.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestQueueCapacities.java index 86feb5bc33f..248831f03d1 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestQueueCapacities.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestQueueCapacities.java @@ -47,7 +47,9 @@ { "AbsoluteMaximumCapacity" }, { "MaxAMResourcePercentage" }, { "ReservedCapacity" }, - { "AbsoluteReservedCapacity" }}); + { "AbsoluteReservedCapacity" }, + { "Weight" }, + { "NormalizedWeight" }}); } public TestQueueCapacities(String suffix) { @@ -105,9 +107,6 @@ private static float executeByName(QueueCapacities obj, String methodName, private void internalTestModifyAndRead(String label) throws Exception { QueueCapacities qc = new QueueCapacities(false); - // First get returns 0 always - Assert.assertEquals(0f, get(qc, suffix, label), 1e-8); - // Set to 1, and check set(qc, suffix, label, 1f); Assert.assertEquals(1f, get(qc, suffix, label), 1e-8); @@ -117,15 +116,19 @@ private void internalTestModifyAndRead(String label) throws Exception { Assert.assertEquals(2f, get(qc, suffix, label), 1e-8); } - void check(int mem, int cpu, Resource res) { - Assert.assertEquals(mem, res.getMemorySize()); - Assert.assertEquals(cpu, res.getVirtualCores()); - } - @Test public void testModifyAndRead() throws Exception { LOG.info("Test - " + suffix); internalTestModifyAndRead(null); internalTestModifyAndRead("label"); } + + @Test + public void testDefaultValues() { + QueueCapacities qc = new QueueCapacities(false); + Assert.assertEquals(-1, qc.getWeight(""), 1e-6); + Assert.assertEquals(-1, qc.getWeight("x"), 1e-6); + Assert.assertEquals(0, qc.getCapacity(""), 1e-6); + Assert.assertEquals(0, qc.getCapacity("x"), 1e-6); + } } \ No newline at end of file diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestQueueParsing.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestQueueParsing.java index c1f48be96a3..236d271104e 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestQueueParsing.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestQueueParsing.java @@ -51,7 +51,7 @@ LoggerFactory.getLogger(TestQueueParsing.class); private static final double DELTA = 0.000001; - + private RMNodeLabelsManager nodeLabelManager; @Before @@ -1143,6 +1143,59 @@ public void testQueueOrderingPolicyUpdatedAfterReinitialize() ServiceOperations.stopQuietly(capacityScheduler); } + @Test(timeout = 60000) + public void testQueueCapacityWithWeight() throws Exception { + YarnConfiguration config = new YarnConfiguration(); + nodeLabelManager = new NullRMNodeLabelsManager(); + nodeLabelManager.init(config); + config.setClass(YarnConfiguration.RM_SCHEDULER, CapacityScheduler.class, + ResourceScheduler.class); + CapacitySchedulerConfiguration conf = + new CapacitySchedulerConfiguration(config); + + // Define top-level queues + conf.setQueues(CapacitySchedulerConfiguration.ROOT, new String[] { "a" }); + conf.setLabeledQueueWeight(CapacitySchedulerConfiguration.ROOT, "x", 100); + conf.setLabeledQueueWeight(CapacitySchedulerConfiguration.ROOT, "y", 100); + conf.setLabeledQueueWeight(CapacitySchedulerConfiguration.ROOT, "z", 100); + + final String A = CapacitySchedulerConfiguration.ROOT + ".a"; + conf.setNonLabeledQueueWeight(A, 100); + conf.setAccessibleNodeLabels(A, ImmutableSet.of("x", "y", "z")); + conf.setLabeledQueueWeight(A, "x", 100); + conf.setLabeledQueueWeight(A, "y", 100); + conf.setLabeledQueueWeight(A, "z", 70); + MockRM rm = null; + try { + rm = new MockRM(conf) { + @Override + public RMNodeLabelsManager createNodeLabelManager() { + return nodeLabelManager; + } + }; + } finally { + IOUtils.closeStream(rm); + } + + verifyQueueAbsCapacity(rm, CapacitySchedulerConfiguration.ROOT, "", 1f); + verifyQueueAbsCapacity(rm, CapacitySchedulerConfiguration.ROOT, "x", 1f); + verifyQueueAbsCapacity(rm, CapacitySchedulerConfiguration.ROOT, "y", 1f); + verifyQueueAbsCapacity(rm, CapacitySchedulerConfiguration.ROOT, "z", 1f); + + verifyQueueAbsCapacity(rm, A, "", 1f); + verifyQueueAbsCapacity(rm, A, "x", 1f); + verifyQueueAbsCapacity(rm, A, "y", 1f); + verifyQueueAbsCapacity(rm, A, "z", 1f); + } + + private void verifyQueueAbsCapacity(MockRM rm, String queuePath, String label, + float expectedAbsCapacity) { + CapacityScheduler cs = (CapacityScheduler) rm.getResourceScheduler(); + CSQueue queue = cs.getQueue(queuePath); + Assert.assertEquals(expectedAbsCapacity, + queue.getQueueCapacities().getAbsoluteCapacity(label), 1e-6); + } + private void checkEqualsToQueueSet(List queues, String[] queueNames) { Set existedQueues = new HashSet<>(); for (CSQueue q : queues) { diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestReservationQueue.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestReservationQueue.java index f6b4f2a31d3..84de7ccb82f 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestReservationQueue.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestReservationQueue.java @@ -28,7 +28,9 @@ import org.apache.hadoop.yarn.conf.YarnConfiguration; import org.apache.hadoop.yarn.server.resourcemanager.RMContext; +import org.apache.hadoop.yarn.server.resourcemanager.scheduler.ResourceLimits; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.SchedulerDynamicEditException; +import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.preemption.PreemptionManager; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.common.QueueEntitlement; import org.apache.hadoop.yarn.util.resource.DefaultResourceCalculator; import org.apache.hadoop.yarn.util.resource.ResourceCalculator; @@ -49,9 +51,10 @@ private final ResourceCalculator resourceCalculator = new DefaultResourceCalculator(); private ReservationQueue autoCreatedLeafQueue; + private PlanQueue planQueue; @Before - public void setup() throws IOException { + public void setup() throws IOException, SchedulerDynamicEditException { // setup a context / conf csConf = new CapacitySchedulerConfiguration(); @@ -66,12 +69,14 @@ public void setup() throws IOException { when(csContext.getClusterResource()).thenReturn( Resources.createResource(100 * 16 * GB, 100 * 32)); when(csContext.getResourceCalculator()).thenReturn(resourceCalculator); + when(csContext.getPreemptionManager()).thenReturn(new PreemptionManager()); RMContext mockRMContext = TestUtils.getMockRMContext(); when(csContext.getRMContext()).thenReturn(mockRMContext); // create a queue - PlanQueue pq = new PlanQueue(csContext, "root", null, null); - autoCreatedLeafQueue = new ReservationQueue(csContext, "a", pq); + planQueue = new PlanQueue(csContext, "root", null, null); + autoCreatedLeafQueue = new ReservationQueue(csContext, "a", planQueue); + planQueue.addChildQueue(autoCreatedLeafQueue); } private void validateAutoCreatedLeafQueue(double capacity) { @@ -83,9 +88,14 @@ private void validateAutoCreatedLeafQueue(double capacity) { @Test public void testAddSubtractCapacity() throws Exception { - // verify that setting, adding, subtracting capacity works autoCreatedLeafQueue.setCapacity(1.0F); + autoCreatedLeafQueue.setMaxCapacity(1.0F); + + planQueue.updateClusterResource( + Resources.createResource(100 * 16 * GB, 100 * 32), + new ResourceLimits(Resources.createResource(100 * 16 * GB, 100 * 32))); + validateAutoCreatedLeafQueue(1); autoCreatedLeafQueue.setEntitlement(new QueueEntitlement(0.9f, 1f)); validateAutoCreatedLeafQueue(0.9); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServices.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServices.java index 76b0796eb29..5785b141144 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServices.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServices.java @@ -1027,7 +1027,7 @@ public void testValidateAndGetSchedulerConfigurationInvalidConfig() Assert.assertEquals(Status.BAD_REQUEST .getStatusCode(), response.getStatus()); Assert.assertTrue(response.getEntity().toString() - .contains("Illegal capacity of 0.5 for children of queue")); + .contains("IOException")); } @Test diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesForCSWithPartitions.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesForCSWithPartitions.java index f5874986867..eb7677f80a8 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesForCSWithPartitions.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesForCSWithPartitions.java @@ -451,8 +451,6 @@ private void verifyQueueBInfoXML(Element queueElem) { Assert.fail("Unexpected partition" + partitionName); } } - } else if (queueChildElem.getTagName().equals("resources")) { - verifyResourceUsageInfoXML(queueChildElem); } } assertEquals("Node Labels are not matching", LABEL_LX, @@ -594,16 +592,12 @@ private void verifySchedulerInfoJson(JSONObject json) for (int i = 0; i < queuesArray.length(); i++) { JSONObject queueJson = queuesArray.getJSONObject(i); String queue = queueJson.getString("queueName"); - - assertEquals("Partition resourceInfo is wrong", 1, - queueJson.getJSONObject("resources") - .getJSONArray(RESOURCE_USAGES_BY_PARTITION).length()); + JSONArray resourceUsageByPartition = queueJson.getJSONObject("resources") + .getJSONArray(RESOURCE_USAGES_BY_PARTITION); JSONObject resourcesJsonObject = queueJson.getJSONObject("resources"); JSONArray partitionsResourcesArray = - resourcesJsonObject.getJSONArray("resourceUsagesByPartition"); - assertEquals("incorrect number of elements", 1, - partitionsResourcesArray.length()); + resourcesJsonObject.getJSONArray(RESOURCE_USAGES_BY_PARTITION); capacitiesJsonObject = queueJson.getJSONObject(CAPACITIES); partitionsCapsArray = @@ -620,6 +614,8 @@ private void verifySchedulerInfoJson(JSONObject json) verifyPartitionCapacityInfoJson(partitionInfo, 30, 0, 50, 30, 0, 50); assertEquals("incorrect number of elements", 7, partitionsResourcesArray.getJSONObject(0).length()); + assertEquals("incorrect number of objects", 1, + resourceUsageByPartition.length()); break; case QUEUE_B: assertEquals("Invalid default Label expression", LABEL_LX, @@ -629,6 +625,8 @@ private void verifySchedulerInfoJson(JSONObject json) verifyAccesibleNodeLabels(queueJson, ImmutableSet.of(LABEL_LX)); assertEquals("incorrect number of partitions", 2, partitionsCapsArray.length()); + assertEquals("incorrect number of objects", 2, + resourceUsageByPartition.length()); for (int j = 0; j < partitionsCapsArray.length(); j++) { partitionInfo = partitionsCapsArray.getJSONObject(j); partitionName = partitionInfo.getString("partitionName");