diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/NodeLabel.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/NodeLabel.java index 1827fb59c91..82c22ac3bc9 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/NodeLabel.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/NodeLabel.java @@ -36,7 +36,7 @@ public static final String DEFAULT_NODE_LABEL_PARTITION = ""; /** - * Node Label expression not set . + * Node Label expression used for display when no node label is set. */ @Private @Unstable diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/nodelabels/CommonNodeLabelsManager.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/nodelabels/CommonNodeLabelsManager.java index 66e945fc2ff..3701261e6c5 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/nodelabels/CommonNodeLabelsManager.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/nodelabels/CommonNodeLabelsManager.java @@ -130,6 +130,11 @@ public Host copy() { } return c; } + + @Override + public String toString() { + return labels + " : " + nms; + } } protected static class Node { @@ -158,6 +163,11 @@ public Node copy() { c.running = running; return c; } + + @Override + public String toString() { + return nodeId + " (" + labels + ")" + resource; + } } private enum NodeLabelUpdateOperation { @@ -561,12 +571,11 @@ private void replaceNodeForLabels(NodeId node, Set oldLabels, } @SuppressWarnings("unchecked") - protected void internalUpdateLabelsOnNodes( + private void internalUpdateLabelsOnNodes( Map> nodeToLabels, NodeLabelUpdateOperation op) throws IOException { // do update labels from nodes - Map> newNMToLabels = - new HashMap>(); + Map> newNMToLabels = new HashMap<>(); Set oldLabels; for (Entry> entry : nodeToLabels.entrySet()) { NodeId nodeId = entry.getKey(); diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/nodelabels/RMNodeLabel.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/nodelabels/RMNodeLabel.java index feeeaf1134e..35f17222a9c 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/nodelabels/RMNodeLabel.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/nodelabels/RMNodeLabel.java @@ -135,4 +135,9 @@ public int hashCode() { return (int) ((((long) labelName.hashCode() << 8) + (resource.hashCode() << 4) + numActiveNMs) % prime); } + + @Override + public String toString() { + return labelName + " " + resource; + } } \ No newline at end of file diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/resource/DefaultResourceCalculator.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/resource/DefaultResourceCalculator.java index bdf60bd9a5b..724021c9508 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/resource/DefaultResourceCalculator.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/resource/DefaultResourceCalculator.java @@ -132,4 +132,9 @@ public boolean fitsIn(Resource cluster, public boolean isAnyMajorResourceZero(Resource resource) { return resource.getMemorySize() == 0f; } + + @Override + public boolean isAnyMajorResourceNonZero(Resource resource) { + return resource.getMemorySize() != 0f; + } } diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/resource/DominantResourceCalculator.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/resource/DominantResourceCalculator.java index 7697e1dfc33..77f164613e9 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/resource/DominantResourceCalculator.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/resource/DominantResourceCalculator.java @@ -244,4 +244,9 @@ public boolean fitsIn(Resource cluster, public boolean isAnyMajorResourceZero(Resource resource) { return resource.getMemorySize() == 0f || resource.getVirtualCores() == 0; } + + @Override + public boolean isAnyMajorResourceNonZero(Resource resource) { + return resource.getMemorySize() != 0f || resource.getVirtualCores() != 0; + } } diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/resource/ResourceCalculator.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/resource/ResourceCalculator.java index 398dac50fa5..21ed2fa79f1 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/resource/ResourceCalculator.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/resource/ResourceCalculator.java @@ -229,11 +229,24 @@ public abstract boolean fitsIn(Resource cluster, Resource smaller, Resource bigger); /** - * Check if resource has any major resource types (which are all NodeManagers - * included) a zero value. + * Return whether the resource has any major resource type values that are + * non-zero. "Major" is determined by the {@link ResourceCalculator} + * instance, e.g. {@link DefaultResourceCalculator} only considers memory + * to be major. * - * @param resource resource - * @return returns true if any resource is zero. + * @param resource the resource to test + * @return returns whether any resource type is zero. */ public abstract boolean isAnyMajorResourceZero(Resource resource); + + /** + * Return whether the resource has any major resource type values that are + * non-zero. "Major" is determined by the {@link ResourceCalculator} + * instance, e.g. {@link DefaultResourceCalculator} only considers memory + * to be major. + * + * @param resource the resource to test + * @return returns whether any resource is non-zero + */ + public abstract boolean isAnyMajorResourceNonZero(Resource resource); } diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/resource/Resources.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/resource/Resources.java index a1d14fdce73..50577d897b7 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/resource/Resources.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/resource/Resources.java @@ -350,9 +350,4 @@ public static Resource componentwiseMax(Resource lhs, Resource rhs) { return createResource(Math.max(lhs.getMemorySize(), rhs.getMemorySize()), Math.max(lhs.getVirtualCores(), rhs.getVirtualCores())); } - - public static boolean isAnyMajorResourceZero(ResourceCalculator rc, - Resource resource) { - return rc.isAnyMajorResourceZero(resource); - } } diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/metrics/TimelineServiceV1Publisher.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/metrics/TimelineServiceV1Publisher.java index 73bb301552e..1020cac6ecc 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/metrics/TimelineServiceV1Publisher.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/metrics/TimelineServiceV1Publisher.java @@ -90,9 +90,9 @@ public void appCreated(RMApp app, long createdTime) { entityInfo.put(ApplicationMetricsConstants.APPLICATION_PRIORITY_INFO, app.getApplicationPriority().getPriority()); entityInfo.put(ApplicationMetricsConstants.AM_NODE_LABEL_EXPRESSION, - app.getAmNodeLabelExpression()); + app.getAmNodeLabelExpressionForDisplay()); entityInfo.put(ApplicationMetricsConstants.APP_NODE_LABEL_EXPRESSION, - app.getAppNodeLabelExpression()); + app.getAppNodeLabelExpressionForDisplay()); if (app.getCallerContext() != null) { if (app.getCallerContext().getContext() != null) { entityInfo.put(ApplicationMetricsConstants.YARN_APP_CALLER_CONTEXT, diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/metrics/TimelineServiceV2Publisher.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/metrics/TimelineServiceV2Publisher.java index 7eaa6e7feb5..48b8bcf6771 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/metrics/TimelineServiceV2Publisher.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/metrics/TimelineServiceV2Publisher.java @@ -121,10 +121,10 @@ public void appCreated(RMApp app, long createdTime) { app.getApplicationPriority().getPriority()); entity.getConfigs().put( ApplicationMetricsConstants.AM_NODE_LABEL_EXPRESSION, - app.getAmNodeLabelExpression()); + app.getAmNodeLabelExpressionForDisplay()); entity.getConfigs().put( ApplicationMetricsConstants.APP_NODE_LABEL_EXPRESSION, - app.getAppNodeLabelExpression()); + app.getAppNodeLabelExpressionForDisplay()); if (app.getCallerContext() != null) { if (app.getCallerContext().getContext() != null) { entityInfo.put(ApplicationMetricsConstants.YARN_APP_CALLER_CONTEXT, diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/monitor/capacity/CapacitySchedulerPreemptionUtils.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/monitor/capacity/CapacitySchedulerPreemptionUtils.java index 0ae3ef01340..5e8b78c7708 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/monitor/capacity/CapacitySchedulerPreemptionUtils.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/monitor/capacity/CapacitySchedulerPreemptionUtils.java @@ -158,7 +158,7 @@ public static boolean tryPreemptContainerAndDeductResToObtain( Resources.none()) && Resources.fitsIn(rc, clusterResource, rmContainer.getAllocatedResource(), totalPreemptionAllowed) - && !Resources.isAnyMajorResourceZero(rc, toObtainByPartition)) { + && !rc.isAnyMajorResourceZero(toObtainByPartition)) { Resources.subtractFrom(toObtainByPartition, rmContainer.getAllocatedResource()); Resources.subtractFrom(totalPreemptionAllowed, diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/monitor/capacity/FifoIntraQueuePreemptionPlugin.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/monitor/capacity/FifoIntraQueuePreemptionPlugin.java index 00ae3dacb9e..2e71f55fe47 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/monitor/capacity/FifoIntraQueuePreemptionPlugin.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/monitor/capacity/FifoIntraQueuePreemptionPlugin.java @@ -273,7 +273,7 @@ private void calculateToBePreemptedResourcePerApp(Resource clusterResource, // Once unallocated resource is 0, we can stop assigning ideal per app. if (Resources.lessThanOrEqual(rc, clusterResource, queueReassignableResource, Resources.none()) - || Resources.isAnyMajorResourceZero(rc, queueReassignableResource)) { + || rc.isAnyMajorResourceZero(queueReassignableResource)) { continue; } diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/nodelabels/RMNodeLabelsManager.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/nodelabels/RMNodeLabelsManager.java index 507f696d057..449ae004908 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/nodelabels/RMNodeLabelsManager.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/nodelabels/RMNodeLabelsManager.java @@ -18,6 +18,7 @@ package org.apache.hadoop.yarn.server.resourcemanager.nodelabels; +import com.google.common.annotations.VisibleForTesting; import java.io.IOException; import java.util.ArrayList; import java.util.Collection; @@ -47,18 +48,105 @@ public class RMNodeLabelsManager extends CommonNodeLabelsManager { protected static class Queue { - protected Set accessibleNodeLabels; - protected Resource resource; + private final Set accessibleNodeLabels; + private final Resource resource; + private boolean any; protected Queue() { accessibleNodeLabels = - Collections.newSetFromMap(new ConcurrentHashMap()); - resource = Resource.newInstance(0, 0); + Collections.newSetFromMap(new ConcurrentHashMap<>()); + resource = Resources.clone(Resources.none()); + any = false; + } + + @Override + public String toString() { + return accessibleNodeLabels + " : " + resource; + } + + /** + * Return whether this queue would accept a host with the given label. + * The {@code labels} parameter is a {@link Set} for historical reasons. + * In practice the set size will always be less than 2. + * + * @param labels the host label as a set + * @return whether the queue accepts the host + */ + public boolean acceptLabels(Set labels) { + // node without any labels can be accessed by any queue + return any || (labels == null) || labels.isEmpty() || + ((labels.size() == 1) && labels.contains(NO_LABEL)) || + !Collections.disjoint(accessibleNodeLabels, labels); + } + + /** + * Return whether this queue will accept all host labels. + * + * @return whether this queue will accept all host labels + */ + @VisibleForTesting + boolean acceptAny() { + return any; + } + + /** + * Return whether this queue explicitly has the given label. If the queue + * has only the {@link RMNodeLabelsManager#ANY} label, this method will + * return true only when the {@code label} parameter is + * {@link RMNodeLabelsManager#ANY}. If the {@code label} parameter is + * {@link RMNodeLabelsManager#NO_LABEL}, then this method will only return + * true if the queue has {@link RMNodeLabelsManager#NO_LABEL} in its set + * of labels. + * + * @param label the label to test + * @return whether the queue has the label + */ + public boolean hasLabel(String label) { + return accessibleNodeLabels.contains(label); + } + + /** + * Set the labels for this queue. + * + * @param labels the labels to set + */ + public void setLabels(Set labels) { + accessibleNodeLabels.clear(); + accessibleNodeLabels.addAll(labels); + any = hasLabel(ANY); + } + + /** + * Add the given resource to this queue's resource total. + * + * @param add the resource to add + */ + public void addResource(Resource add) { + Resources.addTo(resource, add); + } + + /** + * Subtract the given resource from this queue's resource total. The total + * resource cannot be less than 0 for any resource type. + * + * @param subtract the resource to subtract + */ + public void subtractResource(Resource subtract) { + Resources.subtractFromNonNegative(resource, subtract); + } + + /** + * Return the total resources for this queue. + * + * @return the total resources for this queue + */ + public Resource getResource() { + return resource; } } - ConcurrentMap queueCollections = - new ConcurrentHashMap(); + private final ConcurrentMap queueCollections = + new ConcurrentHashMap<>(); private YarnAuthorizationProvider authorizer; private RMContext rmContext = null; @@ -99,8 +187,8 @@ protected void checkRemoveFromClusterNodeLabelsOfQueue( // check if any queue contains this label for (Entry entry : queueCollections.entrySet()) { String queueName = entry.getKey(); - Set queueLabels = entry.getValue().accessibleNodeLabels; - if (queueLabels.contains(label)) { + + if (entry.getValue().hasLabel(label)) { throw new IOException("Cannot remove label=" + label + ", because queue=" + queueName + " is using this label. " + "Please remove label on queue before remove the label"); @@ -320,23 +408,31 @@ public void reinitializeQueueLabels(Map> queueToLabels) { this.queueCollections.clear(); for (Entry> entry : queueToLabels.entrySet()) { - String queue = entry.getKey(); - Queue q = new Queue(); - this.queueCollections.put(queue, q); + addQueue(entry.getKey(), entry.getValue()); + } + } finally { + writeLock.unlock(); + } + } - Set labels = entry.getValue(); - if (labels.contains(ANY)) { - continue; - } + public void addQueue(String queue, Set labels) { + Queue q = new Queue(); + writeLock.lock(); - q.accessibleNodeLabels.addAll(labels); - for (Host host : nodeCollections.values()) { - for (Entry nentry : host.nms.entrySet()) { - NodeId nodeId = nentry.getKey(); - Node nm = nentry.getValue(); - if (nm.running && isNodeUsableByQueue(getLabelsByNode(nodeId), q)) { - Resources.addTo(q.resource, nm.resource); - } + try { + queueCollections.put(queue, q); + + if ((labels != null) && !labels.isEmpty()) { + q.setLabels(labels); + } + + for (Host host : nodeCollections.values()) { + for (Entry nentry : host.nms.entrySet()) { + NodeId nodeId = nentry.getKey(); + Node nm = nentry.getValue(); + + if (nm.running && q.acceptLabels(getLabelsByNode(nodeId))) { + q.addResource(nm.resource); } } } @@ -344,19 +440,29 @@ public void reinitializeQueueLabels(Map> queueToLabels) { writeLock.unlock(); } } - + + /** + * Return the total resources available to this queue according to its node + * labels. If the queue cannot be found, no resources will be returned. + * + * @param queueName the name of the queue whose accessible resources will be + * returned + * @param queueLabels IGNORED + * @param clusterResource IGNORED + * @return the accessible resources for the queue + */ public Resource getQueueResource(String queueName, Set queueLabels, Resource clusterResource) { try { readLock.lock(); - if (queueLabels.contains(ANY)) { - return clusterResource; - } + Queue q = queueCollections.get(queueName); - if (null == q) { + + if (q == null) { return Resources.none(); + } else { + return q.getResource(); } - return q.resource; } finally { readLock.unlock(); } @@ -440,8 +546,7 @@ private void updateResourceMappings(Map before, } // Map used to notify RM - Map> newNodeToLabelsMap = - new HashMap>(); + Map> newNodeToLabelsMap = new HashMap<>(); // traverse all nms for (NodeId nodeId : allNMs) { @@ -456,7 +561,7 @@ private void updateResourceMappings(Map before, // update queues, all queue can access this node for (Queue q : queueCollections.values()) { - Resources.subtractFrom(q.resource, oldNM.resource); + q.subtractResource(oldNM.resource); } } else { // update labels @@ -470,8 +575,8 @@ private void updateResourceMappings(Map before, // update queues, only queue can access this node will be subtract for (Queue q : queueCollections.values()) { - if (isNodeUsableByQueue(oldLabels, q)) { - Resources.subtractFrom(q.resource, oldNM.resource); + if (q.acceptLabels(oldLabels)) { + q.subtractResource(oldNM.resource); } } } @@ -483,7 +588,7 @@ private void updateResourceMappings(Map before, newNodeToLabelsMap.put(nodeId, ImmutableSet.copyOf(newLabels)); - // no label in the past + // no label now if (newLabels.isEmpty()) { // update labels RMNodeLabel label = labelCollections.get(NO_LABEL); @@ -491,7 +596,7 @@ private void updateResourceMappings(Map before, // update queues, all queue can access this node for (Queue q : queueCollections.values()) { - Resources.addTo(q.resource, newNM.resource); + q.addResource(newNM.resource); } } else { // update labels @@ -502,8 +607,8 @@ private void updateResourceMappings(Map before, // update queues, only queue can access this node will be subtract for (Queue q : queueCollections.values()) { - if (isNodeUsableByQueue(newLabels, q)) { - Resources.addTo(q.resource, newNM.resource); + if (q.acceptLabels(newLabels)) { + q.addResource(newNM.resource); } } } @@ -516,7 +621,17 @@ private void updateResourceMappings(Map before, new NodeLabelsUpdateSchedulerEvent(newNodeToLabelsMap)); } } - + + /** + * Get the total amount of resources available from active node managers that + * have the given label. The resources for node managers without a label is + * not included in this total unless the label is + * {@link CommonNodeLabelsManager.NO_LABEL}. + * + * @param label the label for which to return resources + * @param clusterResource IGNORED + * @return the total resources associated with the label + */ public Resource getResourceByLabel(String label, Resource clusterResource) { label = normalizeLabel(label); if (label.equals(NO_LABEL)) { @@ -534,22 +649,6 @@ public Resource getResourceByLabel(String label, Resource clusterResource) { } } - private boolean isNodeUsableByQueue(Set nodeLabels, Queue q) { - // node without any labels can be accessed by any queue - if (nodeLabels == null || nodeLabels.isEmpty() - || (nodeLabels.size() == 1 && nodeLabels.contains(NO_LABEL))) { - return true; - } - - for (String label : nodeLabels) { - if (q.accessibleNodeLabels.contains(label)) { - return true; - } - } - - return false; - } - private Map cloneNodeMap() { Set nodesToCopy = new HashSet(); for (String nodeName : nodeCollections.keySet()) { @@ -587,4 +686,29 @@ public void setRMContext(RMContext rmContext) { readLock.unlock(); } } + + @Override + public String toString() { + StringBuilder out = new StringBuilder(); + + out.append("LABELS:\n"); + for (Entry e : labelCollections.entrySet()) { + out.append("\t").append(e.getKey()).append(": "); + out.append(e.getValue().toString()).append("\n"); + } + + out.append("QUEUES:\n"); + for (Entry e : queueCollections.entrySet()) { + out.append("\t").append(e.getKey()).append(": "); + out.append(e.getValue().toString()).append("\n"); + } + + out.append("NODES:\n"); + for (Entry e : nodeCollections.entrySet()) { + out.append("\t").append(e.getKey()).append(": "); + out.append(e.getValue().toString()).append("\n"); + } + + return out.toString(); + } } diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/RMApp.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/RMApp.java index 93c41b6747c..1958e3d3829 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/RMApp.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/RMApp.java @@ -278,12 +278,30 @@ ApplicationReport createAndGetApplicationReport(String clientUserName, Map getLogAggregationReportsForApp(); LogAggregationStatus getLogAggregationStatusForAppReport(); + /** - * Return the node label expression of the AM container. + * Return the node label of the AM container or null if there is none. + * @return the node label of the AM container */ - String getAmNodeLabelExpression(); + String getAmNodeLabel(); - String getAppNodeLabelExpression(); + /** + * Return the node label of the application or null if there is none. + * @return the node label of the application + */ + String getAppNodeLabel(); + + /** + * Return the node label of the AM container as a string suitable for display. + * @return the node label expression of the AM container + */ + String getAmNodeLabelExpressionForDisplay(); + + /** + * Return the node label of the application as a string suitable for display. + * @return the node label expression of the application + */ + String getAppNodeLabelExpressionForDisplay(); CallerContext getCallerContext(); diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/RMAppImpl.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/RMAppImpl.java index 7526ea3c611..e6916490840 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/RMAppImpl.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/RMAppImpl.java @@ -769,8 +769,8 @@ public ApplicationReport createAndGetApplicationReport(String clientUserName, this.getApplicationPriority()); report.setLogAggregationStatus(logAggregationStatus); report.setUnmanagedApp(submissionContext.getUnmanagedAM()); - report.setAppNodeLabelExpression(getAppNodeLabelExpression()); - report.setAmNodeLabelExpression(getAmNodeLabelExpression()); + report.setAppNodeLabelExpression(getAppNodeLabelExpressionForDisplay()); + report.setAmNodeLabelExpression(getAmNodeLabelExpressionForDisplay()); ApplicationTimeout timeout = ApplicationTimeout .newInstance(ApplicationTimeoutType.LIFETIME, UNLIMITED, UNKNOWN); @@ -1914,31 +1914,52 @@ public String getLogAggregationFailureMessagesForNM(NodeId nodeId) { } @Override - public String getAppNodeLabelExpression() { - String appNodeLabelExpression = - getApplicationSubmissionContext().getNodeLabelExpression(); - appNodeLabelExpression = (appNodeLabelExpression == null) - ? NodeLabel.NODE_LABEL_EXPRESSION_NOT_SET : appNodeLabelExpression; - appNodeLabelExpression = (appNodeLabelExpression.trim().isEmpty()) - ? NodeLabel.DEFAULT_NODE_LABEL_PARTITION : appNodeLabelExpression; - return appNodeLabelExpression; + public String getAppNodeLabel() { + return getApplicationSubmissionContext().getNodeLabelExpression(); } @Override - public String getAmNodeLabelExpression() { + public String getAmNodeLabel() { String amNodeLabelExpression = null; - if (!getApplicationSubmissionContext().getUnmanagedAM()) { + + if (!getApplicationSubmissionContext().getUnmanagedAM() && + (getAMResourceRequests() != null) && + !getAMResourceRequests().isEmpty()) { amNodeLabelExpression = - getAMResourceRequests() != null && !getAMResourceRequests().isEmpty() - ? getAMResourceRequests().get(0).getNodeLabelExpression() : null; - amNodeLabelExpression = (amNodeLabelExpression == null) - ? NodeLabel.NODE_LABEL_EXPRESSION_NOT_SET : amNodeLabelExpression; - amNodeLabelExpression = (amNodeLabelExpression.trim().isEmpty()) - ? NodeLabel.DEFAULT_NODE_LABEL_PARTITION : amNodeLabelExpression; + getAMResourceRequests().get(0).getNodeLabelExpression(); + } + + return amNodeLabelExpression; + } + + @Override + public String getAppNodeLabelExpressionForDisplay() { + return getLabelForDisplay(getAppNodeLabel()); + } + + @Override + public String getAmNodeLabelExpressionForDisplay() { + String amNodeLabelExpression = null; + + if (!getApplicationSubmissionContext().getUnmanagedAM()) { + amNodeLabelExpression = getLabelForDisplay(getAmNodeLabel()); } + return amNodeLabelExpression; } + private String getLabelForDisplay(String label) { + String displayLabel = label; + + if (displayLabel == null) { + displayLabel = NodeLabel.NODE_LABEL_EXPRESSION_NOT_SET; + } else if (displayLabel.isEmpty()) { + displayLabel = NodeLabel.DEFAULT_NODE_LABEL_PARTITION; + } + + return displayLabel; + } + @Override public CallerContext getCallerContext() { return callerContext; diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/Queue.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/Queue.java index d166e5fc568..d5ae7162e7d 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/Queue.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/Queue.java @@ -75,22 +75,16 @@ public void recoverContainer(Resource clusterResource, SchedulerApplicationAttempt schedulerAttempt, RMContainer rmContainer); /** - * Get labels can be accessed of this queue - * labels={*}, means this queue can access any label - * labels={ }, means this queue cannot access any label except node without label - * labels={a, b, c} means this queue can access a or b or c - * @return labels + * Get the set of labels that are available from this queue. + *
    + *
  • labels={*}, means this queue can access any partition
  • + *
  • labels={}, means this queue cannot access any partition except nodes + * without labels
  • + *
  • labels={a, b, c} means this queue can access a or b or c
  • + *
+ * @return the accessible labels */ public Set getAccessibleNodeLabels(); - - /** - * Get default label expression of this queue. If label expression of - * ApplicationSubmissionContext and label expression of Resource Request not - * set, this will be used. - * - * @return default label expression - */ - public String getDefaultNodeLabelExpression(); /** * When new outstanding resource is asked, calling this will increase pending diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/SchedulerApplicationAttempt.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/SchedulerApplicationAttempt.java index c807590c22c..ee7797e1535 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/SchedulerApplicationAttempt.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/SchedulerApplicationAttempt.java @@ -1193,8 +1193,7 @@ public void recordContainerAllocationTime(long value) { } @Private - public boolean hasPendingResourceRequest(ResourceCalculator rc, - String nodePartition, Resource cluster, + public Resource getPendingResourceRequest(String nodePartition, SchedulingMode schedulingMode) { // We need to consider unconfirmed allocations if (schedulingMode == SchedulingMode.IGNORE_PARTITION_EXCLUSIVITY) { @@ -1207,16 +1206,14 @@ public boolean hasPendingResourceRequest(ResourceCalculator rc, // To avoid too many allocation-proposals rejected for non-default // partition allocation if (StringUtils.equals(nodePartition, RMNodeLabelsManager.NO_LABEL)) { + // We use subtract() instead of subtractFrom() because we need to make + // a copy of the original object to avoid modifying it. pending = Resources.subtract(pending, Resources .createResource(unconfirmedAllocatedMem.get(), unconfirmedAllocatedVcores.get())); } - if (Resources.greaterThan(rc, cluster, pending, Resources.none())) { - return true; - } - - return false; + return pending; } /* diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CSQueue.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CSQueue.java index 3a17d1b057d..5ce92c05b58 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CSQueue.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CSQueue.java @@ -37,7 +37,6 @@ import org.apache.hadoop.yarn.api.records.QueueACL; import org.apache.hadoop.yarn.api.records.QueueState; import org.apache.hadoop.yarn.api.records.Resource; -import org.apache.hadoop.yarn.exceptions.InvalidResourceRequestException; import org.apache.hadoop.yarn.security.PrivilegedEntity; import org.apache.hadoop.yarn.server.resourcemanager.rmcontainer.RMContainer; import org.apache.hadoop.yarn.server.resourcemanager.rmcontainer.RMContainerEventType; @@ -46,12 +45,10 @@ import org.apache.hadoop.yarn.server.resourcemanager.scheduler.ResourceUsage; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.SchedulerApplicationAttempt; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.SchedulerQueue; -import org.apache.hadoop.yarn.server.resourcemanager.scheduler.SchedContainerChangeRequest; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.common.ResourceCommitRequest; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.common.fica.FiCaSchedulerApp; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.common.fica.FiCaSchedulerNode; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.placement.PlacementSet; -import org.apache.hadoop.yarn.server.resourcemanager.scheduler.placement.SimplePlacementSet; /** * CSQueue represents a node in the tree of @@ -319,6 +316,16 @@ public void decUsedResource(String nodePartition, Resource resourceToDec, */ public Set getNodeLabelsForQueue(); + + /** + * Get default label expression of this queue. If label expression of + * ApplicationSubmissionContext and label expression of Resource Request not + * set, this will be used. + * + * @return default label expression + */ + public String getDefaultNodeLabelExpression(); + @VisibleForTesting CSAssignment assignContainers(Resource clusterResource, FiCaSchedulerNode node, ResourceLimits resourceLimits, diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacitySchedulerConfiguration.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacitySchedulerConfiguration.java index 3a519ecf5f1..f5a94858989 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacitySchedulerConfiguration.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacitySchedulerConfiguration.java @@ -1116,7 +1116,9 @@ public boolean getPreemptionDisabled(String queue, boolean defaultVal) { } /** - * Get configured node labels in a given queuePath + * Get configured node labels in a given queue path. + * @param queuePath the target queue path + * @return the configured node labels */ public Set getConfiguredNodeLabels(String queuePath) { Set configuredNodeLabels = new HashSet(); diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/allocator/RegularContainerAllocator.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/allocator/RegularContainerAllocator.java index f753d31fdbf..4f645452c41 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/allocator/RegularContainerAllocator.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/allocator/RegularContainerAllocator.java @@ -837,9 +837,12 @@ public CSAssignment assignContainers(Resource clusterResource, FiCaSchedulerNode node = PlacementSetUtils.getSingleNode(ps); if (reservedContainer == null) { + Resource pending = + application.getPendingResourceRequest(ps.getPartition(), + schedulingMode); + // Check if application needs more resource, skip if it doesn't need more. - if (!application.hasPendingResourceRequest(rc, - ps.getPartition(), clusterResource, schedulingMode)) { + if (!rc.isAnyMajorResourceNonZero(pending)) { if (LOG.isDebugEnabled()) { LOG.debug("Skip app_attempt=" + application.getApplicationAttemptId() + ", because it doesn't need more resource, schedulingMode=" diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/AllocationConfiguration.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/AllocationConfiguration.java index 352b58d17b2..c8a5b287ba0 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/AllocationConfiguration.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/AllocationConfiguration.java @@ -92,6 +92,9 @@ private final SchedulingPolicy defaultSchedulingPolicy; + @VisibleForTesting + final Map> accessibleNodeLabels; + // Policy for mapping apps to queues @VisibleForTesting QueuePlacementPolicy placementPolicy; @@ -124,7 +127,8 @@ public AllocationConfiguration(Map minQueueResources, Map> configuredQueues, ReservationQueueConfiguration globalReservationQueueConfig, Set reservableQueues, - Set nonPreemptableQueues) { + Set nonPreemptableQueues, + Map> accessibleNodeLabels) { this.minQueueResources = minQueueResources; this.maxQueueResources = maxQueueResources; this.maxChildQueueResources = maxChildQueueResources; @@ -148,6 +152,7 @@ public AllocationConfiguration(Map minQueueResources, this.placementPolicy = placementPolicy; this.configuredQueues = configuredQueues; this.nonPreemptableQueues = nonPreemptableQueues; + this.accessibleNodeLabels = accessibleNodeLabels; } public AllocationConfiguration(Configuration conf) { @@ -177,6 +182,7 @@ public AllocationConfiguration(Configuration conf) { placementPolicy = QueuePlacementPolicy.fromConfiguration(conf, configuredQueues); nonPreemptableQueues = new HashSet<>(); + accessibleNodeLabels = new HashMap<>(); } /** @@ -317,6 +323,18 @@ SchedulingPolicy getSchedulingPolicy(String queueName) { return (policy == null) ? defaultSchedulingPolicy : policy; } + /** + * Get the set of node labels accessible from the given queue. The return + * value may be null, indicating that no labels were defined for this + * queue. + * + * @param queueName the target queue's name + * @return the set of accessible node labels + */ + public Set getAccessibleNodeLabels(String queueName) { + return accessibleNodeLabels.get(queueName); + } + public SchedulingPolicy getDefaultSchedulingPolicy() { return defaultSchedulingPolicy; } @@ -403,6 +421,7 @@ public void initFSQueue(FSQueue queue){ queue.setMaxRunningApps(getQueueMaxApps(name)); queue.setMaxAMShare(getQueueMaxAMShare(name)); queue.setMaxChildQueueResource(getMaxChildResources(name)); + queue.setAccessibleNodeLabels(getAccessibleNodeLabels(name)); // Set queue metrics. queue.getMetrics().setMinShare(queue.getMinShare()); diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/AllocationFileLoaderService.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/AllocationFileLoaderService.java index 16374e41c84..240bd01c143 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/AllocationFileLoaderService.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/AllocationFileLoaderService.java @@ -46,6 +46,7 @@ import org.apache.hadoop.yarn.security.PrivilegedEntity; import org.apache.hadoop.yarn.security.PrivilegedEntity.EntityType; import org.apache.hadoop.yarn.server.resourcemanager.resource.ResourceWeights; +import org.apache.hadoop.yarn.server.resourcemanager.nodelabels.RMNodeLabelsManager; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.policies.FifoPolicy; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.SchedulerUtils; import org.apache.hadoop.yarn.util.Clock; @@ -243,6 +244,7 @@ public synchronized void reloadAllocations() throws IOException, new HashMap<>(); Set reservableQueues = new HashSet<>(); Set nonPreemptableQueues = new HashSet<>(); + Map> accessibleNodeLabels = new HashMap<>(); int userMaxAppsDefault = Integer.MAX_VALUE; int queueMaxAppsDefault = Integer.MAX_VALUE; ConfigurableResource queueMaxResourcesDefault = @@ -387,7 +389,7 @@ public synchronized void reloadAllocations() throws IOException, queueWeights, queuePolicies, minSharePreemptionTimeouts, fairSharePreemptionTimeouts, fairSharePreemptionThresholds, queueAcls, reservationAcls, configuredQueues, reservableQueues, - nonPreemptableQueues); + nonPreemptableQueues, accessibleNodeLabels); } // Load placement policy and pass it configured queues @@ -437,7 +439,8 @@ public synchronized void reloadAllocations() throws IOException, defaultSchedPolicy, minSharePreemptionTimeouts, fairSharePreemptionTimeouts, fairSharePreemptionThresholds, queueAcls, reservationAcls, newPlacementPolicy, configuredQueues, - globalReservationQueueConfig, reservableQueues, nonPreemptableQueues); + globalReservationQueueConfig, reservableQueues, nonPreemptableQueues, + accessibleNodeLabels); lastSuccessfulReload = clock.getTime(); lastReloadAttemptFailed = false; @@ -464,7 +467,8 @@ private void loadQueue(String parentName, Element element, Map> resAcls, Map> configuredQueues, Set reservableQueues, - Set nonPreemptableQueues) + Set nonPreemptableQueues, + Map> accessibleNodeLabels) throws AllocationConfigurationException { String queueName = FairSchedulerUtilities.trimQueueName( element.getAttribute("name")); @@ -487,14 +491,17 @@ private void loadQueue(String parentName, Element element, Map acls = new HashMap<>(); Map racls = new HashMap<>(); NodeList fields = element.getChildNodes(); + List children = new ArrayList<>(); boolean isLeaf = true; boolean isReservable = false; for (int j = 0; j < fields.getLength(); j++) { Node fieldNode = fields.item(j); + if (!(fieldNode instanceof Element)) { continue; } + Element field = (Element) fieldNode; if ("minResources".equals(field.getTagName())) { String text = ((Text)field.getFirstChild()).getData().trim(); @@ -569,14 +576,34 @@ private void loadQueue(String parentName, Element element, if (!Boolean.parseBoolean(text)) { nonPreemptableQueues.add(queueName); } + } else if ("nodeLabels".equals(field.getTagName())) { + Text data = (Text) field.getFirstChild(); + + if (data != null) { + String[] nodeLabel = data.getData().trim().split(","); + Set nodeLabelSet = new HashSet<>(); + + for (String untrimmedLabel : nodeLabel) { + String label = untrimmedLabel.trim(); + + if (label.equals("-")) { + // "-" is the way to specify "no label" in fair scheduler. + // Capacity scheduler uses " ", which is just asking for + // trouble. The label manager's representation for "no label" + // is "". If we see "-", store it as "". + nodeLabelSet.add(""); + } else if (!label.isEmpty()) { + nodeLabelSet.add(label); + } + } + + accessibleNodeLabels.put(queueName, nodeLabelSet); + } } else if ("queue".endsWith(field.getTagName()) || "pool".equals(field.getTagName())) { - loadQueue(queueName, field, minQueueResources, maxQueueResources, - maxChildQueueResources, queueMaxApps, userMaxApps, queueMaxAMShares, - queueWeights, queuePolicies, minSharePreemptionTimeouts, - fairSharePreemptionTimeouts, fairSharePreemptionThresholds, - queueAcls, resAcls, configuredQueues, reservableQueues, - nonPreemptableQueues); + // Instead of parsing the child queue here, put it in a list to parse + // later so that we can finish parsing this queue first. + children.add(field); isLeaf = false; } } @@ -599,7 +626,7 @@ private void loadQueue(String parentName, Element element, // The root queue defaults to all access for (QueueACL acl : QueueACL.values()) { AccessType accessType = SchedulerUtils.toAccessType(acl); - if (acls.get(accessType) == null){ + if (acls.get(accessType) == null) { AccessControlList defaultAcl = queueName.equals(ROOT) ? EVERYBODY_ACL : NOBODY_ACL; acls.put(accessType, defaultAcl); @@ -609,6 +636,61 @@ private void loadQueue(String parentName, Element element, queueAcls.put(queueName, acls); resAcls.put(queueName, racls); checkMinAndMaxResource(minQueueResources, maxQueueResources, queueName); + + // If we have labels and aren't root, validate the labels + if (accessibleNodeLabels.containsKey(queueName) && (parentName != null)) { + // Check that the node labels are legal. At this point, we may not have + // node labels for all queues, because some may just not specify anything. + // Later they'll inherit from their parents, but not yet. Because of the + // structure of the config file, we are guaranteed that we will have + // parsed our parent queue before we parse our queue. That means that if + // we have no labels set, we can keep walking up the hierarchy until we + // find a node that does have labels or we hit the root. + Set parentLabels = accessibleNodeLabels.get(parentName); + String next = queueName; + + try { + while (parentLabels == null) { + next = next.substring(0, next.lastIndexOf('.')); + parentLabels = accessibleNodeLabels.get(next); + } + } catch (IndexOutOfBoundsException ex) { + // That means we just tried to go past root, so we're done. It's faster + // to catch the exception than check the result of lastIndexOf(). + } + + // If parentLabels is still null, that means there are no labels set + // anywhere above us, and so the whole hierarchy will default to "any", + // and there's nothing to worry about. + if ((parentLabels != null) && !parentLabels.isEmpty() && + !parentLabels.contains(RMNodeLabelsManager.ANY)) { + Set bad = new HashSet<>(accessibleNodeLabels.get(queueName)); + + bad.removeAll(parentLabels); + // No label is always allowed. + bad.remove(RMNodeLabelsManager.NO_LABEL); + + if (!bad.isEmpty()) { + String message = "The fair scheduler configuration file specifies " + + "that " + queueName + " should have the following labels which " + + "are not supported by the parent queue: " + bad + ". Please " + + "remove the labels from " + queueName + " or add them to the " + + "parent queue(s)."; + + throw new AllocationConfigurationException(message); + } + } + } + + // Now that we've finished parsing this queue, parse it's children. + for (Element child : children) { + loadQueue(queueName, child, minQueueResources, maxQueueResources, + maxChildQueueResources, queueMaxApps, userMaxApps, queueMaxAMShares, + queueWeights, queuePolicies, minSharePreemptionTimeouts, + fairSharePreemptionTimeouts, fairSharePreemptionThresholds, + queueAcls, resAcls, configuredQueues, reservableQueues, + nonPreemptableQueues, accessibleNodeLabels); + } } private void checkMinAndMaxResource(Map minResources, diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSAppAttempt.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSAppAttempt.java index 72675b5f787..82628f6aee8 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSAppAttempt.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSAppAttempt.java @@ -44,6 +44,7 @@ import org.apache.hadoop.yarn.server.resourcemanager.RMAuditLogger.AuditConstants; import org.apache.hadoop.yarn.server.resourcemanager.RMContext; import org.apache.hadoop.yarn.server.resourcemanager.resource.ResourceWeights; +import org.apache.hadoop.yarn.server.resourcemanager.nodelabels.RMNodeLabelsManager; import org.apache.hadoop.yarn.server.resourcemanager.rmcontainer.RMContainer; import org.apache.hadoop.yarn.server.resourcemanager.rmcontainer.RMContainerEvent; import org.apache.hadoop.yarn.server.resourcemanager.rmcontainer.RMContainerEventType; @@ -56,6 +57,7 @@ import org.apache.hadoop.yarn.server.resourcemanager.scheduler.SchedulerApplicationAttempt; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.SchedulerNode; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.SchedulerUtils; +import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.SchedulingMode; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.common.PendingAsk; import org.apache.hadoop.yarn.server.scheduler.SchedulerRequestKey; import org.apache.hadoop.yarn.server.utils.BuilderUtils; @@ -228,6 +230,8 @@ private void subtractResourcesOnBlacklistedNodes( /** * Headroom depends on resources in the cluster, current usage of the * queue, queue's fair-share and queue's max-resources. + * + * @return the headroom as a {@link Resource} instance */ @Override public Resource getHeadroom() { @@ -236,9 +240,12 @@ public Resource getHeadroom() { Resource queueFairShare = fsQueue.getFairShare(); Resource queueUsage = fsQueue.getResourceUsage(); - Resource clusterResource = this.scheduler.getClusterResource(); - Resource clusterUsage = this.scheduler.getRootQueueMetrics() - .getAllocatedResources(); + // We don't have a way to track the resources used per partition, so just + // do the math with the cluster resources. The fair share is already capped + // at the partition max, so we should be fine. + Resource clusterResource = scheduler.getClusterResource(); + Resource clusterUsage = + scheduler.getRootQueueMetrics().getAllocatedResources(); Resource clusterAvailableResources = Resources.subtract(clusterResource, clusterUsage); @@ -1350,20 +1357,70 @@ public void updateDemand() { demand = tmpDemand; } + public void nodePartitionUpdated(RMContainer rmContainer, String oldPartition, + String newPartition) { + Resource containerResource = rmContainer.getAllocatedResource(); + this.attemptResourceUsage.decUsed(oldPartition, containerResource); + this.attemptResourceUsage.incUsed(newPartition, containerResource); + + // Update new partition name if container is AM and also update AM resource + if (rmContainer.isAMContainer()) { + this.attemptResourceUsage.decAMUsed(oldPartition, containerResource); + this.attemptResourceUsage.incAMUsed(newPartition, containerResource); + } + } + @Override public Resource assignContainer(FSSchedulerNode node) { + Resource res; + if (isOverAMShareLimit()) { - PendingAsk amAsk = appSchedulingInfo.getNextPendingAsk(); - updateAMDiagnosticMsg(amAsk.getPerAllocationResource(), - " exceeds maximum AM resource allowed)."); - if (LOG.isDebugEnabled()) { - LOG.debug("AM resource request: " + amAsk.getPerAllocationResource() - + " exceeds maximum AM resource allowed, " - + getQueue().dumpState()); + logOverAMShareLimit(); + res = Resources.none(); + } else if (!nodeLabelsMatchContainerLabel(node.getNodeID())) { + res = Resources.none(); + } else { + res = assignContainer(node, false); + } + + return res; + } + + private void logOverAMShareLimit() { + PendingAsk amAsk = appSchedulingInfo.getNextPendingAsk(); + updateAMDiagnosticMsg(amAsk.getPerAllocationResource(), + " exceeds maximum AM resource allowed)."); + if (LOG.isDebugEnabled()) { + LOG.debug("AM resource request: " + amAsk.getPerAllocationResource() + + " exceeds maximum AM resource allowed, " + + getQueue().dumpState()); + } + } + + private boolean nodeLabelsMatchContainerLabel(NodeId nodeId) { + Set labelsOnNode = + scheduler.getLabelsManager().getLabelsOnNode(nodeId); + + // Dig through the pending apps to see if anyone wants this node. + // If we get anything back, go with it, because we support zero-memory + // and zero-CPU requests. + if ((labelsOnNode != null) && !labelsOnNode.isEmpty()) { + for (String label : labelsOnNode) { + Resource pending = getPendingResourceRequest(label, + SchedulingMode.RESPECT_PARTITION_EXCLUSIVITY); + + if (!Resources.isNone(pending)) { + return true; + } } - return Resources.none(); + + return false; + } else { + Resource pending = getPendingResourceRequest(RMNodeLabelsManager.NO_LABEL, + SchedulingMode.RESPECT_PARTITION_EXCLUSIVITY); + + return !Resources.isNone(pending); } - return assignContainer(node, false); } /** diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSLeafQueue.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSLeafQueue.java index ec6fce5e9f3..ce85b48b87f 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSLeafQueue.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSLeafQueue.java @@ -38,6 +38,7 @@ import org.apache.hadoop.yarn.api.records.QueueUserACLInfo; import org.apache.hadoop.yarn.api.records.Resource; import org.apache.hadoop.yarn.server.resourcemanager.resource.ResourceWeights; +import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMApp; import org.apache.hadoop.yarn.server.resourcemanager.rmcontainer.RMContainer; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.ActiveUsersManager; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.SchedulerAppUtils; @@ -590,6 +591,28 @@ boolean isStarved() { return isStarvedForMinShare() || isStarvedForFairShare(); } + /** + * Test whether this queue should accept this app on the basis of node labels. + * A queue with no labels accepts all requests. An app with no labels will + * always be accepted by any queue. + * + * @param app the app to test + * @return true if the queue should accept this app based on node labels + */ + boolean acceptAppNodeLabels(RMApp app) { + if (!acceptAny) { + String appLabel = app.getAppNodeLabel(); + String amLabel = app.getAmNodeLabel(); + + return ((appLabel == null) || appLabel.isEmpty() || + accessibleLabels.contains(appLabel)) && + ((amLabel == null) || amLabel.isEmpty() || + accessibleLabels.contains(amLabel)); + } + + return true; + } + @Override protected void dumpStateInternal(StringBuilder sb) { sb.append("{Name: " + getName() + diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSParentQueue.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSParentQueue.java index a8e53fc26f2..28f935767c1 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSParentQueue.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSParentQueue.java @@ -205,7 +205,7 @@ public Resource assignContainer(FSSchedulerNode node) { try { for (FSQueue child : childQueues) { assigned = child.assignContainer(node); - if (!Resources.equals(assigned, Resources.none())) { + if (!Resources.isNone(assigned)) { break; } } diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSQueue.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSQueue.java index 9be56a374a1..b5e5930888b 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSQueue.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSQueue.java @@ -20,6 +20,7 @@ import java.util.ArrayList; import java.util.Collection; +import java.util.Collections; import java.util.List; import java.util.Set; @@ -30,6 +31,7 @@ import org.apache.hadoop.ipc.Server; import org.apache.hadoop.security.UserGroupInformation; import org.apache.hadoop.yarn.api.records.ApplicationAttemptId; +import org.apache.hadoop.yarn.api.records.NodeId; import org.apache.hadoop.yarn.api.records.Priority; import org.apache.hadoop.yarn.api.records.QueueACL; import org.apache.hadoop.yarn.api.records.QueueInfo; @@ -43,6 +45,7 @@ import org.apache.hadoop.yarn.security.PrivilegedEntity.EntityType; import org.apache.hadoop.yarn.security.YarnAuthorizationProvider; import org.apache.hadoop.yarn.server.resourcemanager.resource.ResourceWeights; +import org.apache.hadoop.yarn.server.resourcemanager.nodelabels.RMNodeLabelsManager; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.Queue; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.SchedulerUtils; import org.apache.hadoop.yarn.util.resource.Resources; @@ -76,6 +79,8 @@ private ConfigurableResource maxShare; protected int maxRunningApps; private ConfigurableResource maxChildQueueResource; + protected Set accessibleLabels; + protected boolean acceptAny = false; // maxAMShare is a value between 0 and 1. protected float maxAMShare; @@ -111,6 +116,7 @@ public final void reinit(boolean recursive) { AllocationConfiguration allocConf = scheduler.getAllocationConfiguration(); allocConf.initFSQueue(this); updatePreemptionVariables(); + updateNodeLabels(); if (recursive) { for (FSQueue child : getChildQueues()) { @@ -119,6 +125,7 @@ public final void reinit(boolean recursive) { } } + @Override public String getName() { return name; } @@ -136,7 +143,7 @@ public FSParentQueue getParent() { return parent; } - public void setPolicy(SchedulingPolicy policy) { + public final void setPolicy(SchedulingPolicy policy) { policy.initialize(scheduler.getContext()); this.policy = policy; } @@ -281,30 +288,55 @@ public FSQueueMetrics getMetrics() { return metrics; } - /** Get the fair share assigned to this Schedulable. */ + /** + * Get the fair share assigned to this Schedulable. + * + * @return the fair share + */ + @Override public Resource getFairShare() { return fairShare; } @Override public void setFairShare(Resource fairShare) { - this.fairShare = fairShare; - metrics.setFairShare(fairShare); + Resource partitionMax = getPartitionResources(); + + this.fairShare = Resources.componentwiseMin(fairShare, partitionMax); + metrics.setFairShare(this.fairShare); + if (LOG.isDebugEnabled()) { - LOG.debug("The updated fairShare for " + getName() + " is " + fairShare); + LOG.debug("The updated fairShare for " + name + " is " + this.fairShare); } } - /** Get the steady fair share assigned to this Schedulable. */ + private Resource getPartitionResources() { + // Even though the last parameter to the getQueueResource() method is + // ignored, if we don't pass in the actual cluster resource, the + // TestFairSchedulerPlanFollower tests will fail. It has to do with the + // way the test mocks the node label manager. + return scheduler.getLabelsManager().getQueueResource(name, null, + scheduler.getClusterResource()); + } + + /** + * Get the steady fair share assigned to this Schedulable. + * + * @return the steady fair share + */ public Resource getSteadyFairShare() { return steadyFairShare; } void setSteadyFairShare(Resource steadyFairShare) { - this.steadyFairShare = steadyFairShare; - metrics.setSteadyFairShare(steadyFairShare); + Resource partitionMax = getPartitionResources(); + + this.steadyFairShare = + Resources.componentwiseMin(steadyFairShare, partitionMax); + metrics.setSteadyFairShare(this.steadyFairShare); } + @Override public boolean hasAccess(QueueACL acl, UserGroupInformation user) { return authorizer.checkPermission( new AccessRequest(queueEntity, user, @@ -397,6 +429,28 @@ private void updatePreemptionVariables() { } /** + * Update node label for this queue. + */ + public void updateNodeLabels() { + accessibleLabels = + scheduler.getAllocationConfiguration().getAccessibleNodeLabels(name); + + if (parent != null) { + if (((accessibleLabels == null) || accessibleLabels.isEmpty())) { + // If no labels were defined, inherit from our parent. For all queues + // but root, the parent is guaranteed to have labels. + accessibleLabels = parent.getAccessibleNodeLabels(); + } + } else if ((accessibleLabels == null) || accessibleLabels.isEmpty()) { + // If we have no parent we're root. If we also have no labels, default + // to ANY. + accessibleLabels = Collections.singleton(RMNodeLabelsManager.ANY); + } + + acceptAny = accessibleLabels.contains(RMNodeLabelsManager.ANY); + } + + /** * Gets the children of this queue, if any. */ public abstract List getChildQueues(); @@ -425,6 +479,17 @@ boolean assignContainerPreCheck(FSSchedulerNode node) { LOG.debug("Assigning container failed on node '" + node.getNodeName() + " because it has reserved containers."); } + + return false; + } else if (!nodeLabelCheck(node.getNodeID())) { + if (LOG.isDebugEnabled()) { + LOG.debug("Assigning container failed on node '" + node.getNodeName() + + " because the node labels don't agree: " + accessibleLabels + + " on the queue v/s " + + scheduler.getLabelsManager().getLabelsOnNode(node.getNodeID()) + + " on the node"); + } + return false; } else if (!Resources.fitsIn(getResourceUsage(), getMaxShare())) { if (LOG.isDebugEnabled()) { @@ -432,10 +497,39 @@ boolean assignContainerPreCheck(FSSchedulerNode node) { + " because queue resource usage is larger than MaxShare: " + dumpState()); } + return false; - } else { - return true; } + + return true; + } + + /** + * Check if the queue's labels allow it to assign containers on this node. + * + * @param nodeId the ID of the node to check + * @return true if the queue is allowed to assign containers on this node + */ + protected boolean nodeLabelCheck(NodeId nodeId) { + // A queue with no label will accept any node + if (!acceptAny) { + Set labelsOnNode = + scheduler.getLabelsManager().getLabelsOnNode(nodeId); + + if ((labelsOnNode == null) || labelsOnNode.isEmpty()) { + return true; + } else { + for (String queueLabel : accessibleLabels) { + if (labelsOnNode.contains(queueLabel)) { + return true; + } + } + + return false; + } + } + + return true; } /** @@ -454,16 +548,18 @@ public String toString() { @Override public Set getAccessibleNodeLabels() { - // TODO, add implementation for FS - return null; + return accessibleLabels; } - - @Override - public String getDefaultNodeLabelExpression() { - // TODO, add implementation for FS - return null; + + /** + * Set the queue's node labels. + * + * @param newLabels the queue's node labels + */ + void setAccessibleNodeLabels(Set newLabels) { + accessibleLabels = newLabels; } - + @Override public void incPendingResource(String nodeLabel, Resource resourceToInc) { } diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FairScheduler.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FairScheduler.java index cbfa9f108e4..cfef2b32b26 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FairScheduler.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FairScheduler.java @@ -52,6 +52,7 @@ import org.apache.hadoop.yarn.server.api.protocolrecords.NMContainerStatus; import org.apache.hadoop.yarn.server.resourcemanager.RMContext; import org.apache.hadoop.yarn.server.resourcemanager.RMCriticalThreadUncaughtExceptionHandler; +import org.apache.hadoop.yarn.server.resourcemanager.nodelabels.RMNodeLabelsManager; import org.apache.hadoop.yarn.server.resourcemanager.recovery.RMStateStore.RMState; import org.apache.hadoop.yarn.server.resourcemanager.reservation.ReservationConstants; import org.apache.hadoop.yarn.server.resourcemanager.resource.ResourceWeights; @@ -80,10 +81,10 @@ import org.apache.hadoop.yarn.server.resourcemanager.scheduler.event.AppRemovedSchedulerEvent; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.event.ContainerExpiredSchedulerEvent; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.event.NodeAddedSchedulerEvent; +import org.apache.hadoop.yarn.server.resourcemanager.scheduler.event.NodeLabelsUpdateSchedulerEvent; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.event.NodeRemovedSchedulerEvent; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.event.NodeResourceUpdateSchedulerEvent; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.event.NodeUpdateSchedulerEvent; - import org.apache.hadoop.yarn.server.resourcemanager.scheduler.event.ReleaseContainerEvent; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.event.SchedulerEvent; import org.apache.hadoop.yarn.server.resourcemanager.security.RMContainerTokenSecretManager; @@ -188,6 +189,8 @@ @VisibleForTesting final MaxRunningAppsEnforcer maxRunningEnforcer; + private RMNodeLabelsManager labelsManager; + private AllocationFileLoaderService allocsLoader; @VisibleForTesting AllocationConfiguration allocConf; @@ -424,6 +427,17 @@ public FairSchedulerEventLog getEventLog() { } /** + * Convenience method for use by other fair scheduler components to get the + * node labels manager. + * + * @return the node labels manager + */ + @VisibleForTesting + public RMNodeLabelsManager getLabelsManager() { + return labelsManager; + } + + /** * Add a new application to the scheduler, with a given id, queue name, and * user. This will accept a new app even if the user or queue is above * configured limits, but the app will not be marked as runnable. @@ -564,8 +578,11 @@ FSLeafQueue assignToQueue(RMApp rmApp, String queueName, String user) { appRejectMsg = "Application rejected by queue placement policy"; } else { queue = queueMgr.getLeafQueue(queueName, true); + if (queue == null) { appRejectMsg = queueName + " is not a leaf queue"; + } else if ((rmApp != null) && !queue.acceptAppNodeLabels(rmApp)) { + appRejectMsg = "Application rejected based on node labels mismatch"; } } } catch (IllegalStateException se) { @@ -723,6 +740,11 @@ private void addNode(List containerReports, usePortForNodeName); nodeTracker.addNode(schedulerNode); + if (labelsManager != null) { + labelsManager.activateNode(node.getNodeID(), + schedulerNode.getTotalResource()); + } + triggerUpdate(); Resource clusterResource = getClusterResource(); @@ -765,6 +787,10 @@ private void removeNode(RMNode rmNode) { SchedulerUtils.LOST_CONTAINER), RMContainerEventType.KILL); } + if (labelsManager != null) { + labelsManager.deactivateNode(rmNode.getNodeID()); + } + nodeTracker.removeNode(nodeId); Resource clusterResource = getClusterResource(); queueMgr.getRootQueue().setSteadyFairShare(clusterResource); @@ -884,6 +910,11 @@ protected void nodeUpdate(RMNode nm) { super.nodeUpdate(nm); FSSchedulerNode fsNode = getFSSchedulerNode(nm.getNodeID()); + + if (labelsManager != null) { + labelsManager.activateNode(nm.getNodeID(), fsNode.getTotalResource()); + } + attemptScheduling(fsNode); long duration = getClock().getTime() - start; @@ -893,6 +924,64 @@ protected void nodeUpdate(RMNode nm) { } } + /** + * Update the labels on a set of nodes. + * + * @param update a mapping of node id to label set + */ + private void nodeLabelsUpdate(Map> update) { + writeLock.lock(); + + try { + for (Entry> entry : update.entrySet()) { + NodeId nodeId = entry.getKey(); + Set labels = entry.getValue(); + FSSchedulerNode node = nodeTracker.getNode(nodeId); + + if (node == null) { + LOG.debug("Received a node labels update event that included a node " + + "that couldn't be found by the node tracker: " + + nodeId.getHost()); + } else { + updateContainerLabels(node, labels); + node.updateLabels(labels); + } + } + } finally { + writeLock.unlock(); + } + } + + /** + * Update the labels for containers running on the node. + * + * @param node the target node + * @param labels the new labels + */ + private void updateContainerLabels(FSSchedulerNode node, Set labels) { + String oldPartition = node.getPartition(); + String newPartition = RMNodeLabelsManager.NO_LABEL; + + // There can only be one label, even though we're being passed a set. + for (String label : labels) { + newPartition = label; + break; + } + + for (RMContainer container : node.getCopiedListOfRunningContainers()) { + FSAppAttempt app = + getApplicationAttempt(container.getApplicationAttemptId()); + + if (app == null) { + LOG.debug("While updating node " + node.getNodeName() + ", " + + "encountered a container that is unknown to the scheduler: " + + container.getApplicationAttemptId()); + } else { + app.nodePartitionUpdated(container, oldPartition, newPartition); + } + } + } + void continuousSchedulingAttempt() throws InterruptedException { long start = getClock().getTime(); List nodeIdList; @@ -1103,6 +1192,16 @@ public void handle(SchedulerEvent event) { NodeUpdateSchedulerEvent nodeUpdatedEvent = (NodeUpdateSchedulerEvent)event; nodeUpdate(nodeUpdatedEvent.getRMNode()); break; + case NODE_LABELS_UPDATE: + if (!(event instanceof NodeLabelsUpdateSchedulerEvent)) { + throw new RuntimeException("Unexpected event type: " + event); + } + NodeLabelsUpdateSchedulerEvent nodeLabelsUpdatedEvent = + (NodeLabelsUpdateSchedulerEvent)event; + + // This event is triggered by a change in the node labels manager + nodeLabelsUpdate(nodeLabelsUpdatedEvent.getUpdatedNodeToLabels()); + break; case APP_ADDED: if (!(event instanceof AppAddedSchedulerEvent)) { throw new RuntimeException("Unexpected event type: " + event); @@ -1248,6 +1347,12 @@ public void recover(RMState state) throws Exception { // NOT IMPLEMENTED } + /** + * Set the {@link RMContext}. + * + * @param rmContext the {@link RMContext} + */ + @Override public void setRMContext(RMContext rmContext) { this.rmContext = rmContext; } @@ -1293,6 +1398,8 @@ private void initScheduler(Configuration conf) throws IOException { eventLog.init(this.conf); allocConf = new AllocationConfiguration(conf); + labelsManager = rmContext.getNodeLabelManager(); + try { queueMgr.initialize(conf); } catch (Exception e) { @@ -1325,6 +1432,8 @@ private void initScheduler(Configuration conf) throws IOException { } catch (Exception e) { throw new IOException("Failed to initialize FairScheduler", e); } + + queueMgr.updateNodeLabels(); } @VisibleForTesting @@ -1636,20 +1745,29 @@ private void verifyMoveDoesNotViolateConstraints(FSAppAttempt app, // maxRunningApps if (cur.getNumRunnableApps() == cur.getMaxRunningApps()) { throw new YarnException("Moving app attempt " + appAttId + " to queue " - + queueName + " would violate queue maxRunningApps constraints on" - + " queue " + cur.getQueueName()); + + queueName + " would violate queue maxRunningApps constraints on " + + "queue " + cur.getQueueName()); } // maxShare if (!Resources.fitsIn(Resources.add(cur.getResourceUsage(), consumption), cur.getMaxShare())) { throw new YarnException("Moving app attempt " + appAttId + " to queue " - + queueName + " would violate queue maxShare constraints on" - + " queue " + cur.getQueueName()); + + queueName + " would violate queue maxShare constraints on " + + "queue " + cur.getQueueName()); } cur = cur.getParent(); } + + // Test that the node labels in the new queue will allow the app + RMApp rmApp = rmContext.getRMApps().get(app.getApplicationId()); + + if (!targetQueue.acceptAppNodeLabels(rmApp)) { + throw new YarnException("Moving app attempt " + appAttId + " to queue " + + queueName + " would violate node label constraints on " + + "queue " + cur.getQueueName()); + } } /** diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/QueueManager.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/QueueManager.java index 8734877f608..27d973f97a0 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/QueueManager.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/QueueManager.java @@ -34,6 +34,7 @@ import org.apache.hadoop.classification.InterfaceAudience.Private; import org.apache.hadoop.classification.InterfaceStability.Unstable; import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.yarn.api.records.Resource; import org.apache.hadoop.yarn.conf.YarnConfiguration; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.policies.FifoPolicy; import org.xml.sax.SAXException; @@ -84,6 +85,20 @@ public void initialize(Configuration conf) throws IOException, } /** + * After the allocation configuration has been loaded, + * register the queues with the node labels manager. + */ + void updateNodeLabels() { + Map> labelsMap = new HashMap<>(); + + for (FSQueue queue : queues.values()) { + labelsMap.put(queue.getName(), queue.getAccessibleNodeLabels()); + } + + scheduler.getLabelsManager().reinitializeQueueLabels(labelsMap); + } + + /** * Get a leaf queue by name, creating it if the create param is true and is necessary. * If the queue is not or can not be a leaf queue, i.e. it already exists as a * parent queue, or one of the parents in its name is already a leaf queue, @@ -93,6 +108,9 @@ public void initialize(Configuration conf) throws IOException, * named "queue1" could be referred to as just "queue1", and a queue named * "queue2" underneath a parent named "parent1" that is underneath the root * could be referred to as just "parent1.queue2". + * @param name the queue name + * @param create whether to create the queue if it doesn't exist + * @return the queue */ public FSLeafQueue getLeafQueue(String name, boolean create) { return getLeafQueue(name, create, true); @@ -135,6 +153,9 @@ public boolean removeLeafQueue(String name) { * named "queue1" could be referred to as just "queue1", and a queue named * "queue2" underneath a parent named "parent1" that is underneath the root * could be referred to as just "parent1.queue2". + * @param name the queue name + * @param create whether to create the queue if it doesn't exist + * @return the queue */ public FSParentQueue getParentQueue(String name, boolean create) { return getParentQueue(name, create, true); @@ -275,10 +296,14 @@ private FSQueue createNewQueues(FSQueueType queueType, String queueName = i.next(); // Check if child policy is allowed - SchedulingPolicy childPolicy = scheduler.getAllocationConfiguration(). - getSchedulingPolicy(queueName); + SchedulingPolicy childPolicy = queueConf.getSchedulingPolicy(queueName); + if (!parent.getPolicy().isChildPolicyAllowed(childPolicy)) { - LOG.error("Can't create queue '" + queueName + "'."); + LOG.error("Can't create queue '" + queueName + "' because the " + + "configured policy (" + childPolicy.getName() + ") is not " + + "allowed by the parent queue's policy (" + + parent.getPolicy().getName() + ")."); + return null; } @@ -300,6 +325,8 @@ private FSQueue createNewQueues(FSQueueType queueType, parent.addChildQueue(queue); setChildResourceLimits(parent, queue, queueConf); queues.put(queue.getName(), queue); + scheduler.getLabelsManager().addQueue(queueName, + queueConf.getAccessibleNodeLabels(queueName)); // If we just created a leaf node, the newParent is null, but that's OK // because we only create a leaf node in the very last iteration. @@ -522,6 +549,8 @@ public void updateAllocationConfiguration(AllocationConfiguration queueConf) { rootQueue.reinit(true); // Update steady fair shares for all queues rootQueue.recomputeSteadyShares(); + + updateNodeLabels(); } /** diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/policies/DominantResourceFairnessPolicy.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/policies/DominantResourceFairnessPolicy.java index 72377b0c096..f8d2f029368 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/policies/DominantResourceFairnessPolicy.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/policies/DominantResourceFairnessPolicy.java @@ -98,11 +98,9 @@ public Resource getHeadroom(Resource queueFairShare, Resource queueUsage, int queueAvailableCPU = Math.max(queueFairShare.getVirtualCores() - queueUsage .getVirtualCores(), 0); - Resource headroom = Resources.createResource( + return Resources.createResource( Math.min(maxAvailable.getMemorySize(), queueAvailableMemory), - Math.min(maxAvailable.getVirtualCores(), - queueAvailableCPU)); - return headroom; + Math.min(maxAvailable.getVirtualCores(), queueAvailableCPU)); } @Override diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fifo/FifoScheduler.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fifo/FifoScheduler.java index 94c7e166ff5..4311a9be4e2 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fifo/FifoScheduler.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fifo/FifoScheduler.java @@ -200,12 +200,6 @@ public void recoverContainer(Resource clusterResource, } @Override - public String getDefaultNodeLabelExpression() { - // TODO add implementation for FIFO scheduler - return null; - } - - @Override public void incPendingResource(String nodeLabel, Resource resourceToInc) { } diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/placement/SimplePlacementSet.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/placement/SimplePlacementSet.java index 48efaa14b27..d5059fe879d 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/placement/SimplePlacementSet.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/placement/SimplePlacementSet.java @@ -22,10 +22,8 @@ import org.apache.hadoop.yarn.api.records.NodeId; import org.apache.hadoop.yarn.api.records.NodeLabel; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.SchedulerNode; -import org.apache.hadoop.yarn.server.resourcemanager.scheduler.common.fica.FiCaSchedulerNode; import java.util.Collections; -import java.util.Iterator; import java.util.Map; /** diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestClientRMService.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestClientRMService.java index 48251c2003f..19032a9f7a1 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestClientRMService.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestClientRMService.java @@ -311,6 +311,8 @@ public void testGetApplicationReport() throws Exception { RecordFactory recordFactory = RecordFactoryProvider.getRecordFactory(null); GetApplicationReportRequest request = recordFactory .newRecordInstance(GetApplicationReportRequest.class); + + // If application has app and am label as null request.setApplicationId(appId1); GetApplicationReportResponse response = rmService.getApplicationReport(request); @@ -319,10 +321,14 @@ public void testGetApplicationReport() throws Exception { report.getApplicationResourceUsageReport(); Assert.assertEquals(10, usageReport.getMemorySeconds()); Assert.assertEquals(3, usageReport.getVcoreSeconds()); - Assert.assertEquals("", report.getAmNodeLabelExpression()); - Assert.assertEquals("", report.getAppNodeLabelExpression()); + Assert.assertEquals("Report contained an AM node label expression when " + + "none was expected", NodeLabel.NODE_LABEL_EXPRESSION_NOT_SET, + report.getAmNodeLabelExpression()); + Assert.assertEquals("Report contained an app node label expression when " + + "none was expected", NodeLabel.NODE_LABEL_EXPRESSION_NOT_SET, + report.getAppNodeLabelExpression()); - // if application has am node label set to blank + // if application has app label as null and am node label set to blank ApplicationId appId2 = getApplicationId(2); when(mockAclsManager.checkAccess(UserGroupInformation.getCurrentUser(), ApplicationAccessType.VIEW_APP, null, appId2)).thenReturn(true); @@ -330,12 +336,14 @@ public void testGetApplicationReport() throws Exception { response = rmService.getApplicationReport(request); report = response.getApplicationReport(); - Assert.assertEquals(NodeLabel.DEFAULT_NODE_LABEL_PARTITION, + Assert.assertEquals("Report contained an AM node label expression when " + + "none was expected", NodeLabel.DEFAULT_NODE_LABEL_PARTITION, report.getAmNodeLabelExpression()); - Assert.assertEquals(NodeLabel.NODE_LABEL_EXPRESSION_NOT_SET, + Assert.assertEquals("Report contained an app node label expression when " + + "none was expected", NodeLabel.NODE_LABEL_EXPRESSION_NOT_SET, report.getAppNodeLabelExpression()); - // if application has am node label set to blank + // if application has app and am node label set to high-mem ApplicationId appId3 = getApplicationId(3); when(mockAclsManager.checkAccess(UserGroupInformation.getCurrentUser(), ApplicationAccessType.VIEW_APP, null, appId3)).thenReturn(true); @@ -1302,7 +1310,7 @@ private void mockRMContext(YarnScheduler yarnScheduler, RMContext rmContext) yarnScheduler); when(rmContext.getRMApps()).thenReturn(apps); when(yarnScheduler.getAppsInQueue(eq("testqueue"))).thenReturn( - getSchedulerApps(apps)); + getSchedulerApps()); ResourceScheduler rs = mock(ResourceScheduler.class); when(rmContext.getScheduler()).thenReturn(rs); } @@ -1323,10 +1331,9 @@ private void mockRMContext(YarnScheduler yarnScheduler, RMContext rmContext) config, "testqueue", 40, 5,"high-mem","high-mem")); return apps; } - - private List getSchedulerApps( - Map apps) { - List schedApps = new ArrayList(); + + private List getSchedulerApps() { + List schedApps = new ArrayList<>(); // Return app IDs for the apps in testqueue (as defined in getRMApps) schedApps.add(ApplicationAttemptId.newInstance(getApplicationId(1), 0)); schedApps.add(ApplicationAttemptId.newInstance(getApplicationId(3), 0)); diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/applicationsmanager/MockAsm.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/applicationsmanager/MockAsm.java index f826631a21d..54183af11ee 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/applicationsmanager/MockAsm.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/applicationsmanager/MockAsm.java @@ -213,12 +213,22 @@ public LogAggregationStatus getLogAggregationStatusForAppReport() { } @Override - public String getAmNodeLabelExpression() { + public String getAmNodeLabel() { throw new UnsupportedOperationException("Not supported yet."); } @Override - public String getAppNodeLabelExpression() { + public String getAppNodeLabel() { + throw new UnsupportedOperationException("Not supported yet."); + } + + @Override + public String getAmNodeLabelExpressionForDisplay() { + throw new UnsupportedOperationException("Not supported yet."); + } + + @Override + public String getAppNodeLabelExpressionForDisplay() { throw new UnsupportedOperationException("Not supported yet."); } diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/metrics/TestSystemMetricsPublisher.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/metrics/TestSystemMetricsPublisher.java index 7005bca6585..d732eb49c4e 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/metrics/TestSystemMetricsPublisher.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/metrics/TestSystemMetricsPublisher.java @@ -106,7 +106,7 @@ public static void tearDown() throws Exception { } } - @Test(timeout = 10000) + @Test//(timeout = 10000) public void testPublishApplicationMetrics() throws Exception { long stateUpdateTimeStamp = System.currentTimeMillis(); for (int i = 1; i <= 2; ++i) { @@ -178,8 +178,9 @@ public void testPublishApplicationMetrics() throws Exception { ApplicationMetricsConstants.APPLICATION_PRIORITY_INFO)); } - Assert.assertEquals(app.getAmNodeLabelExpression(), entity.getOtherInfo() - .get(ApplicationMetricsConstants.AM_NODE_LABEL_EXPRESSION)); + Assert.assertEquals(app.getAmNodeLabelExpressionForDisplay(), + entity.getOtherInfo() + .get(ApplicationMetricsConstants.AM_NODE_LABEL_EXPRESSION)); Assert.assertEquals( app.getApplicationSubmissionContext().getNodeLabelExpression(), @@ -523,12 +524,13 @@ private static RMApp createRMApp(ApplicationId appId) { .thenReturn(Collections.singletonList("java -Xmx1024m")); when(asc.getAMContainerSpec()).thenReturn(containerLaunchContext); when(app.getApplicationSubmissionContext()).thenReturn(asc); - when(app.getAppNodeLabelExpression()).thenCallRealMethod(); + when(app.getAppNodeLabelExpressionForDisplay()).thenCallRealMethod(); + when(app.getAppNodeLabel()).thenCallRealMethod(); ResourceRequest amReq = mock(ResourceRequest.class); when(amReq.getNodeLabelExpression()).thenReturn("high-mem"); when(app.getAMResourceRequests()) .thenReturn(Collections.singletonList(amReq)); - when(app.getAmNodeLabelExpression()).thenCallRealMethod(); + when(app.getAmNodeLabelExpressionForDisplay()).thenCallRealMethod(); when(app.getApplicationPriority()).thenReturn(Priority.newInstance(10)); when(app.getCallerContext()) .thenReturn(new CallerContext.Builder("context").build()); diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/nodelabels/NullRMNodeLabelsManager.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/nodelabels/NullRMNodeLabelsManager.java index bb0b45f6917..defca582eaf 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/nodelabels/NullRMNodeLabelsManager.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/nodelabels/NullRMNodeLabelsManager.java @@ -87,7 +87,7 @@ protected void stopDispatcher() { @Override protected void serviceInit(Configuration conf) throws Exception { - // always enable node labels while using MemoryRMNodeLabelsManager + // always enable node labels while using NullRMNodeLabelsManager conf.setBoolean(YarnConfiguration.NODE_LABELS_ENABLED, true); super.serviceInit(conf); } diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/nodelabels/TestRMNodeLabelsManager.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/nodelabels/TestRMNodeLabelsManager.java index 1da6f93f664..4964b82a1bd 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/nodelabels/TestRMNodeLabelsManager.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/nodelabels/TestRMNodeLabelsManager.java @@ -28,6 +28,7 @@ import java.io.IOException; import java.util.Arrays; import java.util.Collection; +import java.util.Collections; import java.util.HashMap; import java.util.List; import java.util.Map; @@ -61,12 +62,15 @@ import com.google.common.collect.ImmutableSet; public class TestRMNodeLabelsManager extends NodeLabelTestBase { - private final Resource EMPTY_RESOURCE = Resource.newInstance(0, 0); - private final Resource SMALL_RESOURCE = Resource.newInstance(100, 0); - private final Resource LARGE_NODE = Resource.newInstance(1000, 0); + private static final Resource EMPTY_RESOURCE = Resource.newInstance(0, 0); + private static final Resource SMALL_RESOURCE = Resource.newInstance(100, 0); + private static final Resource LARGE_NODE = Resource.newInstance(1000, 0); + private static final Resource ONE = Resource.newInstance(1024, 1); + private static final Resource TWO = Resource.newInstance(2048, 2); + private static final Resource THREE = Resource.newInstance(3072, 3); + private static final Resource FOUR = Resource.newInstance(4096, 4); - NullRMNodeLabelsManager mgr = null; - RMNodeLabelsManager lmgr = null; + RMNodeLabelsManager mgr = null; boolean checkQueueCall = false; @Before public void before() { @@ -228,8 +232,6 @@ public void testGetLabelResource() throws Exception { @Test(timeout=5000) public void testGetQueueResource() throws Exception { - Resource clusterResource = Resource.newInstance(9999, 1); - /* * Node->Labels: * host1 : red @@ -264,6 +266,8 @@ public void testGetQueueResource() throws Exception { mgr.reinitializeQueueLabels(queueToLabels); + Resource clusterResource = Resource.newInstance(400, 0); + // check resource Assert.assertEquals(Resources.multiply(SMALL_RESOURCE, 3), mgr.getQueueResource("Q1", q1Label, clusterResource)); @@ -309,6 +313,7 @@ public void testGetQueueResource() throws Exception { mgr.deactivateNode(NodeId.newInstance("host1", 1)); mgr.deactivateNode(NodeId.newInstance("host3", 1)); mgr.activateNode(NodeId.newInstance("host3", 1), SMALL_RESOURCE); + clusterResource = Resource.newInstance(300, 0); // check resource Assert.assertEquals(Resources.multiply(SMALL_RESOURCE, 2), @@ -367,6 +372,7 @@ public void testGetQueueResource() throws Exception { mgr.activateNode(NodeId.newInstance("host3", 2), SMALL_RESOURCE); mgr.activateNode(NodeId.newInstance("host3", 3), SMALL_RESOURCE); mgr.activateNode(NodeId.newInstance("host4", 2), SMALL_RESOURCE); + clusterResource = Resource.newInstance(600, 0); // check resource Assert.assertEquals(Resources.multiply(SMALL_RESOURCE, 3), @@ -390,6 +396,7 @@ public void testGetQueueResource() throws Exception { mgr.deactivateNode(NodeId.newInstance("host3", 3)); mgr.deactivateNode(NodeId.newInstance("host4", 2)); mgr.deactivateNode(NodeId.newInstance("host4", 1)); + clusterResource = Resource.newInstance(300, 0); // check resource Assert.assertEquals(Resources.multiply(SMALL_RESOURCE, 1), @@ -610,7 +617,7 @@ public void testPullRMNodeLabelsInfo() throws IOException { @Test(timeout = 60000) public void testcheckRemoveFromClusterNodeLabelsOfQueue() throws Exception { - lmgr = new RMNodeLabelsManager(); + RMNodeLabelsManager lmgr = new RMNodeLabelsManager(); Configuration conf = new Configuration(); File tempDir = File.createTempFile("nlb", ".tmp"); tempDir.delete(); @@ -621,34 +628,186 @@ public void testcheckRemoveFromClusterNodeLabelsOfQueue() throws Exception { conf.setBoolean(YarnConfiguration.NODE_LABELS_ENABLED, true); conf.set(YarnConfiguration.RM_SCHEDULER, "org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacityScheduler"); - Configuration withQueueLabels = getConfigurationWithQueueLabels(conf); - MockRM rm = initRM(conf); + MockRM rm = initRM(conf, lmgr); lmgr.addToCluserNodeLabels(toSet(NodeLabel.newInstance("x", false))); lmgr.removeFromClusterNodeLabels(Arrays.asList(new String[] { "x" })); lmgr.addToCluserNodeLabelsWithDefaultExclusivity(toSet("x")); rm.stop(); - class TestRMLabelManger extends RMNodeLabelsManager { + + RMNodeLabelsManager testMgr = new RMNodeLabelsManager() { @Override protected void checkRemoveFromClusterNodeLabelsOfQueue( Collection labelsToRemove) throws IOException { checkQueueCall = true; // Do nothing } - } - lmgr = new TestRMLabelManger(); - MockRM rm2 = initRM(withQueueLabels); - Assert.assertFalse( - "checkRemoveFromClusterNodeLabelsOfQueue should not be called" - + "on recovery", + }; + Configuration withQueueLabels = getConfigurationWithQueueLabels(conf); + MockRM rm2 = initRM(withQueueLabels, testMgr); + Assert.assertFalse("checkRemoveFromClusterNodeLabelsOfQueue should not " + + "be called on recovery", checkQueueCall); - lmgr.removeFromClusterNodeLabels(Arrays.asList(new String[] { "x" })); - Assert - .assertTrue("checkRemoveFromClusterNodeLabelsOfQueue should be called " - + "since its not recovery", checkQueueCall); + testMgr.removeFromClusterNodeLabels(Arrays.asList(new String[] {"x"})); + Assert.assertTrue("checkRemoveFromClusterNodeLabelsOfQueue should " + + "be called since its not recovery", checkQueueCall); rm2.stop(); } - private MockRM initRM(Configuration conf) { + @Test + public void testAddQueue() throws IOException { + initNodesAndQueues(); + + mgr.addQueue("queue0", Collections.EMPTY_SET); + Assert.assertEquals("Queue with no labels should bind to one host", + ONE, mgr.getQueueResource("queue0", null, null)); + + mgr.addQueue("queue1", ImmutableSet.of("label1")); + Assert.assertEquals("Queue with one label should bind to two hosts", + TWO, mgr.getQueueResource("queue1", null, null)); + + mgr.addQueue("queue2", ImmutableSet.of("label1", "label2")); + Assert.assertEquals("Queue with two labels should bind to three hosts", + THREE, mgr.getQueueResource("queue2", null, null)); + + mgr.addQueue("queue3", ImmutableSet.of(RMNodeLabelsManager.ANY)); + Assert.assertEquals("Queue with all labels should bind to all hosts", + FOUR, mgr.getQueueResource("queue3", null, null)); + } + + @Test + public void testReinitializeQueueLabels() throws IOException { + initNodesAndQueues(); + + Map> map = new HashMap<>(); + + map.put("queue0", Collections.EMPTY_SET); + map.put("queue1", ImmutableSet.of("label1")); + map.put("queue2", ImmutableSet.of("label1", "label2")); + map.put("queue3", ImmutableSet.of(RMNodeLabelsManager.ANY)); + + mgr.reinitializeQueueLabels(map); + + Assert.assertEquals("Queue with no labels should bind to one host", + ONE, mgr.getQueueResource("queue0", null, null)); + Assert.assertEquals("Queue with one label should bind to two hosts", + TWO, mgr.getQueueResource("queue1", null, null)); + Assert.assertEquals("Queue with two labels should bind to three hosts", + THREE, mgr.getQueueResource("queue2", null, null)); + Assert.assertEquals("Queue with all labels should bind to all hosts", + FOUR, mgr.getQueueResource("queue3", null, null)); + } + + private void initNodesAndQueues() throws IOException { + Map> map = new HashMap<>(); + Set labels = ImmutableSet.of("label1", "label2", "label3"); + NodeId node1 = NodeId.newInstance("127.0.0.1", 9000); + NodeId node2 = NodeId.newInstance("127.0.0.1", 9001); + NodeId node3 = NodeId.newInstance("127.0.0.2", 9000); + NodeId node4 = NodeId.newInstance("127.0.0.2", 9001); + + map.put(node1, Collections.singleton("label1")); + map.put(node2, Collections.singleton("label2")); + map.put(node3, Collections.singleton("label3")); + + mgr.addToCluserNodeLabelsWithDefaultExclusivity(labels); + mgr.activateNode(node1, ONE); + mgr.activateNode(node2, ONE); + mgr.activateNode(node3, ONE); + mgr.activateNode(node4, ONE); + mgr.addLabelsToNode(map); + + Assert.assertEquals("label1 should only have a single host associated " + + "with it", ONE, mgr.getResourceByLabel("label1", null)); + Assert.assertEquals("label2 should only have a single host associated " + + "with it", ONE, mgr.getResourceByLabel("label2", null)); + Assert.assertEquals("label3 should only have a single host associated " + + "with it", ONE, mgr.getResourceByLabel("label3", null)); + Assert.assertEquals(" should only have a single host associated " + + "with it", + ONE, mgr.getResourceByLabel(RMNodeLabelsManager.NO_LABEL, null)); + } + + @Test public void testQueue() { + RMNodeLabelsManager.Queue q = new RMNodeLabelsManager.Queue(); + + Assert.assertEquals("A fresh queue should have no resources", + Resources.none(), q.getResource()); + Assert.assertFalse("A fresh queue should not have the ANY label", + q.acceptAny()); + + q.addResource(ONE); + Assert.assertEquals("Added 1GB,1vcore to 0GB,0vcores", + ONE, q.getResource()); + q.addResource(TWO); + Assert.assertEquals("Added 2GB,2vcores to 1GB,1vcore", + THREE, q.getResource()); + q.subtractResource(THREE); + Assert.assertEquals("Subtracted 3GB,3vcores from 3GB,3vcore", + Resources.none(), q.getResource()); + q.subtractResource(ONE); + Assert.assertEquals("Subtracted 1GB,1vcores from 0GB,0vcore", + Resources.none(), q.getResource()); + + q.setLabels(ImmutableSet.of(RMNodeLabelsManager.NO_LABEL)); + Assert.assertTrue("Queue with didn't report having ", + q.hasLabel(RMNodeLabelsManager.NO_LABEL)); + Assert.assertTrue("Queue with doesn't accept ", + q.acceptLabels(ImmutableSet.of(RMNodeLabelsManager.NO_LABEL))); + Assert.assertFalse("Queue with accepts label1", + q.acceptLabels(ImmutableSet.of("label1"))); + Assert.assertFalse("Queue with accepts label1 or label2", + q.acceptLabels(ImmutableSet.of("label1", "label2"))); + Assert.assertFalse("Queue with accepts any", q.acceptAny()); + + q.setLabels(ImmutableSet.of("label1")); + Assert.assertTrue("Queue with 1 label didn't report having that label", + q.hasLabel("label1")); + Assert.assertFalse("Queue without reported having ", + q.hasLabel(RMNodeLabelsManager.NO_LABEL)); + Assert.assertTrue("Queue with label1 doesn't accept label1", + q.acceptLabels(ImmutableSet.of("label1"))); + Assert.assertTrue("Queue with label1 doesn't accept ", + q.acceptLabels(ImmutableSet.of(RMNodeLabelsManager.NO_LABEL))); + Assert.assertTrue("Queue with label1 doesn't accept label1 or label2", + q.acceptLabels(ImmutableSet.of("label1", "label2"))); + Assert.assertFalse("Queue with label1 accepts any", q.acceptAny()); + + q.setLabels(ImmutableSet.of("label1", "label3")); + Assert.assertTrue("Queue with label1 and label3 didn't report having " + + "label1", q.hasLabel("label1")); + Assert.assertTrue("Queue with label1 and label3 didn't report having " + + "label3", q.hasLabel("label3")); + Assert.assertFalse("Queue without reported having ", + q.hasLabel(RMNodeLabelsManager.NO_LABEL)); + Assert.assertTrue("Queue with label1 and label3 doesn't accept label1", + q.acceptLabels(ImmutableSet.of("label1"))); + Assert.assertTrue("Queue with label1 and label3 doesn't accept label3", + q.acceptLabels(ImmutableSet.of("label3"))); + Assert.assertTrue("Queue with label1 and label3 doesn't accept ", + q.acceptLabels(ImmutableSet.of(RMNodeLabelsManager.NO_LABEL))); + Assert.assertTrue("Queue with label1 and label3 doesn't accept label1 " + + "or label2", q.acceptLabels(ImmutableSet.of("label1", "label2"))); + Assert.assertFalse("Queue with label1 and label3 accepts any", + q.acceptAny()); + + q.setLabels(ImmutableSet.of(RMNodeLabelsManager.ANY)); + Assert.assertTrue("Queue with didn't report having " + + "", q.hasLabel(RMNodeLabelsManager.ANY)); + Assert.assertFalse("Queue without reported having ", + q.hasLabel(RMNodeLabelsManager.NO_LABEL)); + Assert.assertTrue("Queue with doesn't accept label1", + q.acceptLabels(ImmutableSet.of("label1"))); + Assert.assertTrue("Queue with doesn't accept label4", + q.acceptLabels(ImmutableSet.of("label4"))); + Assert.assertTrue("Queue with doesn't accept ", + q.acceptLabels(ImmutableSet.of(RMNodeLabelsManager.NO_LABEL))); + Assert.assertTrue("Queue with doesn't accept label1 " + + "or label2", q.acceptLabels(ImmutableSet.of("label1", "label2"))); + Assert.assertTrue("Queue with doesn't accept any", + q.acceptAny()); + } + + private MockRM initRM(Configuration conf, RMNodeLabelsManager lmgr) { MockRM rm = new MockRM(conf) { @Override public RMNodeLabelsManager createNodeLabelManager() { @@ -708,7 +867,7 @@ public void testLabelsToNodesOnNodeActiveDeactive() throws Exception { @Test(timeout = 60000) public void testBackwardsCompatableMirror() throws Exception { - lmgr = new RMNodeLabelsManager(); + RMNodeLabelsManager lmgr = new RMNodeLabelsManager(); Configuration conf = new Configuration(); File tempDir = File.createTempFile("nlb", ".tmp"); tempDir.delete(); @@ -739,7 +898,7 @@ public void testBackwardsCompatableMirror() throws Exception { "org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacityScheduler"); Configuration withQueueLabels = getConfigurationWithQueueLabels(conf); - MockRM rm = initRM(withQueueLabels); + MockRM rm = initRM(withQueueLabels, lmgr); Set labelNames = lmgr.getClusterNodeLabelNames(); Map> labeledNodes = lmgr.getLabelsToNodes(); diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/MockRMApp.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/MockRMApp.java index 39a7f995ab6..91590175735 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/MockRMApp.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/MockRMApp.java @@ -294,12 +294,22 @@ public LogAggregationStatus getLogAggregationStatusForAppReport() { } @Override - public String getAmNodeLabelExpression() { + public String getAmNodeLabel() { return null; } @Override - public String getAppNodeLabelExpression() { + public String getAppNodeLabel() { + return null; + } + + @Override + public String getAmNodeLabelExpressionForDisplay() { + return null; + } + + @Override + public String getAppNodeLabelExpressionForDisplay() { return null; } diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestApplicationLimits.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestApplicationLimits.java index 8aca235f8aa..f1047b3d4d0 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestApplicationLimits.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestApplicationLimits.java @@ -578,7 +578,7 @@ public void testHeadroom() throws Exception { thenReturn(Resources.createResource(16*GB)); when(csContext.getResourceCalculator()).thenReturn(resourceCalculator); when(csContext.getRMContext()).thenReturn(rmContext); - + // Say cluster has 100 nodes of 16G each Resource clusterResource = Resources.createResource(100 * 16 * GB); when(csContext.getClusterResource()).thenReturn(clusterResource); diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacitySchedulerAsyncScheduling.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacitySchedulerAsyncScheduling.java index 0c3130dc2f3..ca92ac5ed13 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacitySchedulerAsyncScheduling.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacitySchedulerAsyncScheduling.java @@ -70,8 +70,6 @@ private YarnConfiguration conf; - RMNodeLabelsManager mgr; - @Before public void setUp() throws Exception { conf = new YarnConfiguration(); @@ -79,8 +77,6 @@ public void setUp() throws Exception { ResourceScheduler.class); conf.setBoolean( CapacitySchedulerConfiguration.SCHEDULE_ASYNCHRONOUSLY_ENABLE, true); - mgr = new NullRMNodeLabelsManager(); - mgr.init(conf); } @Test(timeout = 300000) @@ -155,17 +151,17 @@ public RMNodeLabelsManager createNodeLabelManager() { waitTime -= 50; } - Assert.assertEquals( - rm.getResourceScheduler().getRootQueueMetrics().getAllocatedMB(), - totalAsked); + Assert.assertEquals("Didn't see expected amount of allocated resources", + totalAsked, + rm.getResourceScheduler().getRootQueueMetrics().getAllocatedMB()); // Wait for another 2 sec to make sure we will not allocate more than // required waitTime = 2000; // ms while (waitTime > 0) { - Assert.assertEquals( - rm.getResourceScheduler().getRootQueueMetrics().getAllocatedMB(), - totalAsked); + Assert.assertEquals("Saw more than expected amount of allocated " + + "resources", totalAsked, + rm.getResourceScheduler().getRootQueueMetrics().getAllocatedMB()); waitTime -= 50; Thread.sleep(50); } diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestContainerAllocation.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestContainerAllocation.java index b1ca72a88b5..aeaa60c93bb 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestContainerAllocation.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestContainerAllocation.java @@ -99,7 +99,7 @@ public void testExcessReservationThanNodeManagerCapacity() throws Exception { // wait.. int waitCount = 20; - int size = rm.getRMContext().getRMNodes().size(); + int size; while ((size = rm.getRMContext().getRMNodes().size()) != 2 && waitCount-- > 0) { LOG.info("Waiting for node managers to register : " + size); diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestUtils.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestUtils.java index 4bc5127e9da..483725b24a7 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestUtils.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestUtils.java @@ -63,11 +63,17 @@ import org.apache.hadoop.yarn.server.utils.BuilderUtils; import org.apache.hadoop.yarn.util.resource.DefaultResourceCalculator; import org.apache.hadoop.yarn.util.resource.Resources; -import org.mockito.invocation.InvocationOnMock; -import org.mockito.stubbing.Answer; import com.google.common.collect.Sets; import org.apache.hadoop.yarn.event.Event; +import java.io.IOException; +import java.util.Map; +import java.util.Set; + +import static org.mockito.Mockito.doReturn; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.spy; +import static org.mockito.Mockito.when; public class TestUtils { private static final Log LOG = LogFactory.getLog(TestUtils.class); @@ -101,33 +107,28 @@ public void register(Class eventType, new ContainerAllocationExpirer(nullDispatcher); Configuration conf = new Configuration(); - RMApplicationHistoryWriter writer = mock(RMApplicationHistoryWriter.class); RMContextImpl rmContext = new RMContextImpl(nullDispatcher, cae, null, null, null, new AMRMTokenSecretManager(conf, null), new RMContainerTokenSecretManager(conf), new NMTokenSecretManagerInRM(conf), new ClientToAMTokenSecretManagerInRM()); - RMNodeLabelsManager nlm = mock(RMNodeLabelsManager.class); - when( - nlm.getQueueResource(any(String.class), any(Set.class), - any(Resource.class))).thenAnswer(new Answer() { + // This was originally a mock, but calling reinitializeQueues() on the mock + // was a no-op for some reason. Instead it's now an anonymous inner class. + RMNodeLabelsManager nlm = new RMNodeLabelsManager() { @Override - public Resource answer(InvocationOnMock invocation) throws Throwable { - Object[] args = invocation.getArguments(); - return (Resource) args[2]; + public Resource getResourceByLabel(String label, + Resource clusterResource) { + return clusterResource; } - }); - - when(nlm.getResourceByLabel(any(String.class), any(Resource.class))) - .thenAnswer(new Answer() { - @Override public Resource answer(InvocationOnMock invocation) - throws Throwable { - Object[] args = invocation.getArguments(); - return (Resource) args[1]; - } - }); + @Override + public Resource getQueueResource(String queueName, + Set queueLabels, Resource clusterResource) { + return clusterResource; + } + }; + nlm.init(conf); rmContext.setNodeLabelManager(nlm); rmContext.setSystemMetricsPublisher(mock(SystemMetricsPublisher.class)); rmContext.setRMApplicationHistoryWriter(mock(RMApplicationHistoryWriter.class)); diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FairSchedulerTestBase.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FairSchedulerTestBase.java index af4e1dd32a0..478699a6d93 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FairSchedulerTestBase.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FairSchedulerTestBase.java @@ -95,6 +95,7 @@ public Configuration createConfiguration() { conf.setBoolean(FairSchedulerConfiguration.ASSIGN_MULTIPLE, false); conf.setLong(FairSchedulerConfiguration.UPDATE_INTERVAL_MS, 10); conf.setFloat(FairSchedulerConfiguration.PREEMPTION_THRESHOLD, 0f); + conf.setBoolean(YarnConfiguration.NODE_LABELS_ENABLED, true); conf.setFloat( FairSchedulerConfiguration @@ -118,7 +119,15 @@ protected ResourceRequest createResourceRequest( protected ResourceRequest createResourceRequest( int memory, int vcores, String host, int priority, int numContainers, boolean relaxLocality) { - ResourceRequest request = recordFactory.newRecordInstance(ResourceRequest.class); + return createResourceRequest(memory, vcores, host, priority, numContainers, + relaxLocality, RMNodeLabelsManager.NO_LABEL); + } + + protected ResourceRequest createResourceRequest( + int memory, int vcores, String host, int priority, int numContainers, + boolean relaxLocality, String label) { + ResourceRequest request = + recordFactory.newRecordInstance(ResourceRequest.class); request.setCapability(BuilderUtils.newResource(memory, vcores)); request.setResourceName(host); request.setNumContainers(numContainers); @@ -126,7 +135,8 @@ protected ResourceRequest createResourceRequest( prio.setPriority(priority); request.setPriority(prio); request.setRelaxLocality(relaxLocality); - request.setNodeLabelExpression(RMNodeLabelsManager.NO_LABEL); + request.setNodeLabelExpression(label); + return request; } @@ -163,6 +173,13 @@ protected ApplicationAttemptId createSchedulingRequest( protected ApplicationAttemptId createSchedulingRequest( int memory, int vcores, String queueId, String userId, int numContainers, int priority) { + return createSchedulingRequest(memory, vcores, queueId, userId, + numContainers, priority, RMNodeLabelsManager.NO_LABEL); + } + + protected ApplicationAttemptId createSchedulingRequest( + int memory, int vcores, String queueId, String userId, int numContainers, + int priority, String label) { ApplicationAttemptId id = createAppAttemptId(this.APP_ID++, this.ATTEMPT_ID++); scheduler.addApplication(id.getApplicationId(), queueId, userId, false); // This conditional is for testAclSubmitApplication where app is rejected @@ -170,9 +187,9 @@ protected ApplicationAttemptId createSchedulingRequest( if (scheduler.getSchedulerApplications().containsKey(id.getApplicationId())) { scheduler.addApplicationAttempt(id, false, false); } - List ask = new ArrayList(); + List ask = new ArrayList<>(); ResourceRequest request = createResourceRequest(memory, vcores, ResourceRequest.ANY, - priority, numContainers, true); + priority, numContainers, true, label); ask.add(request); RMApp rmApp = mock(RMApp.class); @@ -188,8 +205,8 @@ protected ApplicationAttemptId createSchedulingRequest( resourceManager.getRMContext().getRMApps() .put(id.getApplicationId(), rmApp); - scheduler.allocate(id, ask, new ArrayList(), - null, null, NULL_UPDATE_REQUESTS); + scheduler.allocate(id, ask, new ArrayList<>(), null, null, + NULL_UPDATE_REQUESTS); scheduler.update(); return id; } diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestAllocationFileLoaderService.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestAllocationFileLoaderService.java index 67b46f99398..2fa76bfc54e 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestAllocationFileLoaderService.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestAllocationFileLoaderService.java @@ -22,10 +22,21 @@ import java.io.File; import java.io.FileOutputStream; import java.io.FileWriter; +import java.io.IOException; import java.io.OutputStreamWriter; import java.io.PrintWriter; import java.nio.charset.StandardCharsets; +import java.nio.file.FileSystems; +import java.nio.file.FileVisitResult; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.SimpleFileVisitor; +import java.nio.file.attribute.BasicFileAttributes; +import java.util.Arrays; +import java.util.Collections; +import java.util.HashSet; import java.util.List; +import java.util.Set; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.yarn.api.records.QueueACL; @@ -36,16 +47,50 @@ import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.policies.FairSharePolicy; import org.apache.hadoop.yarn.util.ControlledClock; import org.apache.hadoop.yarn.util.resource.Resources; +import org.junit.AfterClass; +import org.junit.BeforeClass; import org.junit.Test; public class TestAllocationFileLoaderService { final static String TEST_DIR = new File(System.getProperty("test.build.data", - "/tmp")).getAbsolutePath(); + "/tmp"), "alloctest").getAbsolutePath(); final static String ALLOC_FILE = new File(TEST_DIR, "test-queues").getAbsolutePath(); + private static final Set ANY = Collections.singleton("*"); + + @BeforeClass + public static void setup() throws IOException { + new File(TEST_DIR).mkdirs(); + } + + @AfterClass + public static void cleanup() throws IOException { + // Remove the tmp dir + Files.walkFileTree(FileSystems.getDefault().getPath(TEST_DIR), + new SimpleFileVisitor() { + @Override + public FileVisitResult visitFile(Path file, BasicFileAttributes attrs) + throws IOException { + Files.delete(file); + return FileVisitResult.CONTINUE; + } + + @Override + public FileVisitResult postVisitDirectory(Path dir, IOException exc) + throws IOException { + if (exc == null) { + Files.delete(dir); + return FileVisitResult.CONTINUE; + } else { + throw exc; + } + } + }); + } + @Test public void testGetAllocationFileFromClasspath() { Configuration conf = new Configuration(); @@ -98,6 +143,10 @@ public void testReload() throws Exception { .contains("root.queueA")); assertTrue(allocConf.getConfiguredQueues().get(FSQueueType.LEAF) .contains("root.queueB")); + assertNull("root.queueA has node labels even though none were assigned", + allocConf.getAccessibleNodeLabels("root.queueA")); + assertNull("root.queueB has node labels even though none were assigned", + allocConf.getAccessibleNodeLabels("root.queueB")); confHolder.allocConf = null; @@ -107,6 +156,7 @@ public void testReload() throws Exception { out.println(""); out.println(" "); out.println(" 3"); + out.println(" label1"); out.println(" "); out.println(" "); out.println(" "); @@ -142,6 +192,9 @@ public void testReload() throws Exception { .size()); assertTrue(allocConf.getConfiguredQueues().get(FSQueueType.LEAF) .contains("root.queueB")); + assertEquals("root.queueB doesn't have the assigned label", + Collections.singleton("label1"), + allocConf.getAccessibleNodeLabels("root.queueB")); } @Test @@ -164,25 +217,30 @@ public void testAllocationFileParsing() throws Exception { out.println("5120mb,110vcores"); out.println("alice,bob admins"); out.println("fair"); + out.println(""); out.println(""); // Give queue C no minimum out.println(""); out.println("5120mb,0vcores"); out.println("alice,bob admins"); + out.println("-"); out.println(""); // Give queue D a limit of 3 running apps and 0.4f maxAMShare out.println(""); out.println("3"); out.println("0.4"); + out.println(" "); out.println(""); // Give queue E a preemption timeout of one minute out.println(""); out.println("60"); + out.println("label1"); out.println(""); // Make queue F a parent queue without configured leaf queues using the // 'type' attribute out.println(""); out.println("2048mb,64vcores"); + out.println("*"); out.println(""); // Create hierarchical queues G,H, with different min/fair share preemption // timeouts and preemption thresholds. Also add a child default to make sure @@ -192,10 +250,12 @@ public void testAllocationFileParsing() throws Exception { out.println("120"); out.println("50"); out.println("0.6"); + out.println("label1,label2"); out.println(" "); out.println(" 180"); out.println(" 40"); out.println(" 0.7"); + out.println(" label2"); out.println(" "); out.println(""); // Set default limit of apps per queue to 15 @@ -296,13 +356,36 @@ public void testAllocationFileParsing() throws Exception { assertEquals(10, queueConf.getUserMaxApps("user1")); assertEquals(5, queueConf.getUserMaxApps("user2")); - assertEquals(.5f, queueConf.getQueueMaxAMShare("root." + YarnConfiguration.DEFAULT_QUEUE_NAME), 0.01); + assertEquals(.5f, queueConf.getQueueMaxAMShare("root." + + YarnConfiguration.DEFAULT_QUEUE_NAME), 0.01); assertEquals(.5f, queueConf.getQueueMaxAMShare("root.queueA"), 0.01); assertEquals(.5f, queueConf.getQueueMaxAMShare("root.queueB"), 0.01); assertEquals(.5f, queueConf.getQueueMaxAMShare("root.queueC"), 0.01); assertEquals(.4f, queueConf.getQueueMaxAMShare("root.queueD"), 0.01); assertEquals(.5f, queueConf.getQueueMaxAMShare("root.queueE"), 0.01); + assertNull("root.queueA has node labels even though none were assigned", + queueConf.getAccessibleNodeLabels("root.queueA")); + assertNull("root.queueB has node labels even though none were assigned", + queueConf.getAccessibleNodeLabels("root.queueB")); + assertEquals("root.queueC doesn't have the expected labels", + new HashSet<>(Arrays.asList("")), + queueConf.getAccessibleNodeLabels("root.queueC")); + assertEquals("root.queueD doesn't have the expected labels", + new HashSet<>(), queueConf.getAccessibleNodeLabels("root.queueD")); + assertEquals("root.queueE doesn't have the expected labels", + new HashSet<>(Arrays.asList("label1")), + queueConf.getAccessibleNodeLabels("root.queueE")); + assertEquals("root.queueF doesn't have the expected labels", + new HashSet<>(Arrays.asList("*")), + queueConf.getAccessibleNodeLabels("root.queueF")); + assertEquals("root.queueG doesn't have the expected labels", + new HashSet<>(Arrays.asList("label1", "label2")), + queueConf.getAccessibleNodeLabels("root.queueG")); + assertEquals("root.queueH doesn't have the expected labels", + new HashSet<>(Arrays.asList("label2")), + queueConf.getAccessibleNodeLabels("root.queueG.queueH")); + // Root should get * ACL assertEquals("*", queueConf.getQueueAcl("root", QueueACL.ADMINISTER_QUEUE).getAclString()); @@ -387,7 +470,7 @@ public void testAllocationFileParsing() throws Exception { assertEquals(DominantResourceFairnessPolicy.NAME, queueConf.getSchedulingPolicy("root.newqueue").getName()); } - + @Test public void testBackwardsCompatibleAllocationFileParsing() throws Exception { Configuration conf = new Configuration(); @@ -832,6 +915,338 @@ public void testReservableCannotBeCombinedWithDynamicUserQueue() allocLoader.reloadAllocations(); } + @Test //(timeout = 5000) + public void testNodeLabelConfiguration() throws Exception { + Configuration conf = new Configuration(); + conf.set(FairSchedulerConfiguration.ALLOCATION_FILE, ALLOC_FILE); + + try (PrintWriter out = new PrintWriter(new FileWriter(ALLOC_FILE))) { + out.println(""); + out.println(""); + out.println(""); + out.println(" "); + out.println(" *"); + out.println(" "); + out.println(""); + out.println(""); + } + + AllocationFileLoaderService allocLoader = new AllocationFileLoaderService(); + ReloadListener confHolder = new ReloadListener(); + allocLoader.init(conf); + allocLoader.setReloadListener(confHolder); + allocLoader.reloadAllocations(); + + assertNull("root has node labels even though none were assigned", + confHolder.allocConf.getAccessibleNodeLabels("root")); + assertEquals("root.queueA doesn't have the expected labels", ANY, + confHolder.allocConf.getAccessibleNodeLabels("root.queueA")); + + try (PrintWriter out = new PrintWriter(new FileWriter(ALLOC_FILE))) { + out.println(""); + out.println(""); + out.println(""); + out.println(" label1 , label2 "); + out.println(""); + out.println(""); + } + + allocLoader.reloadAllocations(); + + assertEquals("root doesn't have the expected labels", + new HashSet<>(Arrays.asList("label1", "label2")), + confHolder.allocConf.getAccessibleNodeLabels("root")); + + try (PrintWriter out = new PrintWriter(new FileWriter(ALLOC_FILE))) { + out.println(""); + out.println(""); + out.println(""); + out.println(" label1 "); + out.println(""); + out.println(""); + } + + allocLoader.reloadAllocations(); + + assertEquals("root doesn't have the expected labels", + new HashSet<>(Arrays.asList("label1")), + confHolder.allocConf.getAccessibleNodeLabels("root")); + + try (PrintWriter out = new PrintWriter(new FileWriter(ALLOC_FILE))) { + out.println(""); + out.println(""); + out.println(""); + out.println(" "); + out.println(" -"); + out.println(" "); + out.println(""); + out.println(""); + } + + allocLoader.reloadAllocations(); + + assertNull("root has node labels even though none were assigned", + confHolder.allocConf.getAccessibleNodeLabels("root")); + assertEquals("root.queueA doesn't have the expected labels", + new HashSet<>(Arrays.asList("")), + confHolder.allocConf.getAccessibleNodeLabels("root.queueA")); + + try (PrintWriter out = new PrintWriter(new FileWriter(ALLOC_FILE))) { + out.println(""); + out.println(""); + out.println(""); + out.println(" "); + out.println(" label1"); + out.println(" "); + out.println(""); + out.println(""); + } + + allocLoader.reloadAllocations(); + + assertNull("root has node labels even though none were assigned", + confHolder.allocConf.getAccessibleNodeLabels("root")); + assertEquals("root.queueA doesn't have the expected labels", + new HashSet<>(Arrays.asList("label1")), + confHolder.allocConf.getAccessibleNodeLabels("root.queueA")); + + try (PrintWriter out = new PrintWriter(new FileWriter(ALLOC_FILE))) { + out.println(""); + out.println(""); + out.println(""); + out.println(" "); + out.println(" -"); + out.println(" "); + out.println(" *"); + out.println(""); + out.println(""); + } + + allocLoader.reloadAllocations(); + + assertEquals("root doesn't have the expected labels", ANY, + confHolder.allocConf.getAccessibleNodeLabels("root")); + assertEquals("root.queueA doesn't have the expected labels", + new HashSet<>(Arrays.asList("")), + confHolder.allocConf.getAccessibleNodeLabels("root.queueA")); + + try (PrintWriter out = new PrintWriter(new FileWriter(ALLOC_FILE))) { + out.println(""); + out.println(""); + out.println(""); + out.println(" "); + out.println(" -"); + out.println(" "); + out.println(" label1"); + out.println(""); + out.println(""); + } + + allocLoader.reloadAllocations(); + + assertEquals("root doesn't have the expected labels", + new HashSet<>(Arrays.asList("label1")), + confHolder.allocConf.getAccessibleNodeLabels("root")); + assertEquals("root.queueA doesn't have the expected labels", + new HashSet<>(Arrays.asList("")), + confHolder.allocConf.getAccessibleNodeLabels("root.queueA")); + + try (PrintWriter out = new PrintWriter(new FileWriter(ALLOC_FILE))) { + out.println(""); + out.println(""); + out.println(""); + out.println(" label1"); + out.println(" "); + out.println(" label1"); + out.println(" "); + out.println(""); + out.println(""); + } + + allocLoader.reloadAllocations(); + + assertEquals("root doesn't have the expected labels", + new HashSet<>(Arrays.asList("label1")), + confHolder.allocConf.getAccessibleNodeLabels("root")); + assertEquals("root.queueA doesn't have the expected labels", + new HashSet<>(Arrays.asList("label1")), + confHolder.allocConf.getAccessibleNodeLabels("root.queueA")); + + try (PrintWriter out = new PrintWriter(new FileWriter(ALLOC_FILE))) { + out.println(""); + out.println(""); + out.println(""); + out.println(" "); + out.println(" label1,label2"); + out.println(" "); + out.println(" label1"); + out.println(" "); + out.println(" "); + out.println(""); + out.println(""); + } + + allocLoader.reloadAllocations(); + + assertNull("root has node labels even though none were assigned", + confHolder.allocConf.getAccessibleNodeLabels("root")); + assertEquals("root.queueA doesn't have the expected labels", + new HashSet<>(Arrays.asList("label1", "label2")), + confHolder.allocConf.getAccessibleNodeLabels("root.queueA")); + assertEquals("root.queueA.queueB doesn't have the expected labels", + new HashSet<>(Arrays.asList("label1")), + confHolder.allocConf.getAccessibleNodeLabels("root.queueA.queueB")); + + try (PrintWriter out = new PrintWriter(new FileWriter(ALLOC_FILE))) { + out.println(""); + out.println(""); + out.println(""); + out.println(" "); + out.println(" "); + out.println(" label1"); + out.println(" "); + out.println(" label1,label2"); + out.println(" "); + out.println(" *"); + out.println(""); + out.println(""); + } + + allocLoader.reloadAllocations(); + + assertEquals("root doesn't have the expected labels", ANY, + confHolder.allocConf.getAccessibleNodeLabels("root")); + assertEquals("root.queueA doesn't have the expected labels", + new HashSet<>(Arrays.asList("label1", "label2")), + confHolder.allocConf.getAccessibleNodeLabels("root.queueA")); + assertEquals("root.queueA.queueB doesn't have the expected labels", + new HashSet<>(Arrays.asList("label1")), + confHolder.allocConf.getAccessibleNodeLabels("root.queueA.queueB")); + + try (PrintWriter out = new PrintWriter(new FileWriter(ALLOC_FILE))) { + out.println(""); + out.println(""); + out.println(""); + out.println(" label2"); + out.println(" "); + out.println(" label1"); + out.println(" "); + out.println(""); + out.println(""); + } + + try { + allocLoader.reloadAllocations(); + fail("Expected an exception when the child queue has a label not present " + + "on the parent"); + } catch (AllocationConfigurationException ex) { + } + + try (PrintWriter out = new PrintWriter(new FileWriter(ALLOC_FILE))) { + out.println(""); + out.println(""); + out.println(""); + out.println(" "); + out.println(" label1"); + out.println(" "); + out.println(" -"); + out.println(""); + out.println(""); + } + + try { + allocLoader.reloadAllocations(); + fail("Expected an exception when the child queue has a label not present " + + "on the parent"); + } catch (AllocationConfigurationException ex) { + } + + try (PrintWriter out = new PrintWriter(new FileWriter(ALLOC_FILE))) { + out.println(""); + out.println(""); + out.println(""); + out.println(" label2"); + out.println(" "); + out.println(" "); + out.println(" label1"); + out.println(" "); + out.println(" "); + out.println(""); + out.println(""); + } + + try { + allocLoader.reloadAllocations(); + fail("Expected an exception when the child queue has a label not present " + + "on the parent"); + } catch (AllocationConfigurationException ex) { + } + + try (PrintWriter out = new PrintWriter(new FileWriter(ALLOC_FILE))) { + out.println(""); + out.println(""); + out.println(""); + out.println(" "); + out.println(" "); + out.println(" label1"); + out.println(" "); + out.println(" "); + out.println(" -"); + out.println(""); + out.println(""); + } + + try { + allocLoader.reloadAllocations(); + fail("Expected an exception when the child queue has a label not present " + + "on the parent"); + } catch (AllocationConfigurationException ex) { + } + + try (PrintWriter out = new PrintWriter(new FileWriter(ALLOC_FILE))) { + out.println(""); + out.println(""); + out.println(""); + out.println(" "); + out.println(" label2"); + out.println(" "); + out.println(" label1"); + out.println(" "); + out.println(" "); + out.println(" *"); + out.println(""); + out.println(""); + } + + try { + allocLoader.reloadAllocations(); + fail("Expected an exception when the child queue has a label not present " + + "on the parent"); + } catch (AllocationConfigurationException ex) { + } + + try (PrintWriter out = new PrintWriter(new FileWriter(ALLOC_FILE))) { + out.println(""); + out.println(""); + out.println(""); + out.println(" "); + out.println(" "); + out.println(" label1"); + out.println(" "); + out.println(" label2"); + out.println(" "); + out.println(""); + out.println(""); + } + + try { + allocLoader.reloadAllocations(); + fail("Expected an exception when the child queue has a label not present " + + "on the parent"); + } catch (AllocationConfigurationException ex) { + } + } + private class ReloadListener implements AllocationFileLoaderService.Listener { public AllocationConfiguration allocConf; diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFSLeafQueue.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFSLeafQueue.java index 4a738ca07fb..8216325392c 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFSLeafQueue.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFSLeafQueue.java @@ -73,7 +73,7 @@ public void teardown() { } @Test - public void testUpdateDemand() { + public void testUpdateDemand() throws AllocationConfigurationException { conf.set(FairSchedulerConfiguration.ASSIGN_MULTIPLE, "false"); resourceManager = new MockRM(conf); resourceManager.start(); @@ -142,7 +142,7 @@ public void test() throws Exception { } @Test - public void testConcurrentAccess() { + public void testConcurrentAccess() throws AllocationConfigurationException { conf.set(FairSchedulerConfiguration.ASSIGN_MULTIPLE, "false"); resourceManager = new MockRM(conf); resourceManager.start(); diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFSNodeLabel.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFSNodeLabel.java new file mode 100644 index 00000000000..a3691ffa2f3 --- /dev/null +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFSNodeLabel.java @@ -0,0 +1,237 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair; + +import java.io.File; +import java.io.FileWriter; +import java.io.IOException; +import java.io.PrintWriter; +import java.util.HashMap; +import java.util.HashSet; +import java.util.Map; +import java.util.Set; + +import static org.junit.Assert.assertTrue; + +import org.apache.hadoop.metrics2.lib.DefaultMetricsSystem; +import org.apache.hadoop.yarn.api.records.ApplicationAttemptId; +import org.apache.hadoop.yarn.api.records.NodeId; +import org.apache.hadoop.yarn.api.records.NodeLabel; +import org.apache.hadoop.yarn.conf.YarnConfiguration; +import org.apache.hadoop.yarn.event.AsyncDispatcher; +import org.apache.hadoop.yarn.server.resourcemanager.MockNodes; +import org.apache.hadoop.yarn.server.resourcemanager.RMContext; +import org.apache.hadoop.yarn.server.resourcemanager.ResourceManager; +import org.apache.hadoop.yarn.server.resourcemanager.nodelabels.RMNodeLabelsManager; +import org.apache.hadoop.yarn.server.resourcemanager.rmapp.attempt.RMAppAttemptState; +import org.apache.hadoop.yarn.server.resourcemanager.rmcontainer.RMContainer; +import org.apache.hadoop.yarn.server.resourcemanager.rmnode.RMNode; +import org.apache.hadoop.yarn.server.resourcemanager.scheduler.QueueMetrics; +import org.apache.hadoop.yarn.server.resourcemanager.scheduler.event.AppAttemptRemovedSchedulerEvent; +import org.apache.hadoop.yarn.server.resourcemanager.scheduler.event.NodeAddedSchedulerEvent; +import org.apache.hadoop.yarn.server.resourcemanager.scheduler.event.NodeUpdateSchedulerEvent; +import org.apache.hadoop.yarn.util.resource.Resources; +import org.junit.After; +import org.junit.Before; +import org.junit.Test; + + +@SuppressWarnings("unchecked") +public class TestFSNodeLabel extends FairSchedulerTestBase { + private final static String ALLOC_FILE = + new File(TEST_DIR, "test-queues").getAbsolutePath(); + + @Before + public void setUp() throws IOException { + scheduler = new FairScheduler(); + conf = createConfiguration(); + conf.setBoolean(YarnConfiguration.NODE_LABELS_ENABLED, true); + resourceManager = new ResourceManager(); + resourceManager.init(conf); + + RMContext context = resourceManager.getRMContext(); + // TODO: This test should really be using MockRM. For now starting stuff + // that is needed at a bare minimum. + ((AsyncDispatcher)context.getDispatcher()).start(); + context.getStateStore().start(); + + // to initialize the master key + context.getContainerTokenSecretManager().rollMasterKey(); + + scheduler.setRMContext(context); + } + + @After + public void tearDown() { + if (scheduler != null) { + scheduler.stop(); + scheduler = null; + } + if (resourceManager != null) { + resourceManager.stop(); + resourceManager = null; + } + QueueMetrics.clearQueueMetrics(); + DefaultMetricsSystem.shutdown(); + } + + @Test + public void testAssignmentWithNodeLabel() throws Exception { + conf.set(FairSchedulerConfiguration.ALLOCATION_FILE, ALLOC_FILE); + + PrintWriter out = new PrintWriter(new FileWriter(ALLOC_FILE)); + out.println(""); + out.println(""); + out.println(""); + out.println(""); + out.println("1024mb,0vcores"); + out.println(""); + out.println(""); + out.println("1024mb,0vcores"); + out.println("label2"); + out.println(""); + out.println(""); + out.println("label3,label4"); + out.println(""); + out.println(""); + out.println(""); + out.close(); + + scheduler.init(conf); + scheduler.start(); + scheduler.reinitialize(conf, resourceManager.getRMContext()); + + // Create 3 nodes + RMNode node1 = + MockNodes.newNodeInfo(1, Resources.createResource(4 * 1024, 4), 1, + "127.0.0.1"); + NodeAddedSchedulerEvent nodeEvent1 = new NodeAddedSchedulerEvent(node1); + scheduler.handle(nodeEvent1); + RMNode node2 = + MockNodes.newNodeInfo(1, Resources.createResource(4 * 1024, 4), 1, + "127.0.0.2"); + NodeAddedSchedulerEvent nodeEvent2 = new NodeAddedSchedulerEvent(node2); + scheduler.handle(nodeEvent2); + RMNode node3 = + MockNodes.newNodeInfo(1, Resources.createResource(4 * 1024, 4), 1, + "127.0.0.3"); + NodeAddedSchedulerEvent nodeEvent3 = new NodeAddedSchedulerEvent(node3); + scheduler.handle(nodeEvent3); + // add node label + Set clusterLabel = new HashSet<>(); + clusterLabel.add(NodeLabel.newInstance("label1")); + clusterLabel.add(NodeLabel.newInstance("label2")); + clusterLabel.add(NodeLabel.newInstance("label3")); + scheduler.getLabelsManager().addToCluserNodeLabels(clusterLabel); + Map> nodeLabelMap = new HashMap<>(); + // node1: label1 + // node2: label2, label3 + // node3: + Set nodeLabel1 = new HashSet<>(); + nodeLabel1.add(RMNodeLabelsManager.NO_LABEL); + Set nodeLabel2 = new HashSet<>(); + nodeLabel2.add("label2"); + Set nodeLabel3 = new HashSet<>(); + nodeLabel3.add("label3"); + nodeLabelMap.put(node1.getNodeID(), nodeLabel1); + nodeLabelMap.put(node2.getNodeID(), nodeLabel2); + nodeLabelMap.put(node3.getNodeID(), nodeLabel3); + scheduler.getLabelsManager().addLabelsToNode(nodeLabelMap); + + //case1 : app submitted into queue without node label could allocated + // on each node + ApplicationAttemptId appAttId1 = createSchedulingRequest(3074, 1, + "root.queueA", "user1", 3, 3); + NodeUpdateSchedulerEvent nodeUpdate11 = new NodeUpdateSchedulerEvent(node1); + NodeUpdateSchedulerEvent nodeUpdate12 = new NodeUpdateSchedulerEvent(node2); + NodeUpdateSchedulerEvent nodeUpdate13 = new NodeUpdateSchedulerEvent(node3); + + scheduler.update(); + scheduler.handle(nodeUpdate11); + scheduler.handle(nodeUpdate12); + scheduler.handle(nodeUpdate13); + + // app2 will allocated on node1 + for (RMContainer container : + scheduler.getSchedulerApp(appAttId1).getLiveContainers()) { + assertTrue("Request with no label did not run on a host with no label", + container.getAllocatedNode().equals(node1.getNodeID())); + } + + AppAttemptRemovedSchedulerEvent appRemovedEvent1 = + new AppAttemptRemovedSchedulerEvent( + appAttId1, RMAppAttemptState.FINISHED, false); + + scheduler.handle(appRemovedEvent1); + scheduler.update(); + + //case2 : app submitted into queueB could only be allocated on node1 + // now super app finished + ApplicationAttemptId appAttId2 = createSchedulingRequest(1024, 1, + "root.queueB", "user1", 3, 3, "label2"); + NodeUpdateSchedulerEvent nodeUpdate21 = new NodeUpdateSchedulerEvent(node1); + NodeUpdateSchedulerEvent nodeUpdate22 = new NodeUpdateSchedulerEvent(node2); + NodeUpdateSchedulerEvent nodeUpdate23 = new NodeUpdateSchedulerEvent(node3); + + scheduler.update(); + scheduler.handle(nodeUpdate21); + scheduler.handle(nodeUpdate22); + scheduler.handle(nodeUpdate23); + + // app2 will allocated on node2 + for (RMContainer container : + scheduler.getSchedulerApp(appAttId2).getLiveContainers()) { + assertTrue("Request for label2 did not run on a host with label2", + container.getAllocatedNode().equals(node2.getNodeID())); + } + + AppAttemptRemovedSchedulerEvent appRemovedEvent2 = + new AppAttemptRemovedSchedulerEvent( + appAttId2, RMAppAttemptState.FINISHED, false); + + scheduler.handle(appRemovedEvent2); + scheduler.update(); + + //case3 : app submitted into queueC could be allocated on node3 + // now super app finished + ApplicationAttemptId appAttId3 = createSchedulingRequest(1024, 1, + "root.queueC", "user1", 3, 3, "label3"); + NodeUpdateSchedulerEvent nodeUpdate31 = new NodeUpdateSchedulerEvent(node1); + NodeUpdateSchedulerEvent nodeUpdate32 = new NodeUpdateSchedulerEvent(node2); + NodeUpdateSchedulerEvent nodeUpdate33 = new NodeUpdateSchedulerEvent(node3); + + scheduler.update(); + scheduler.handle(nodeUpdate31); + scheduler.handle(nodeUpdate32); + scheduler.handle(nodeUpdate33); + + // app3 will allocated on node2 + for (RMContainer container : + scheduler.getSchedulerApp(appAttId3).getLiveContainers()) { + assertTrue("Request for label3 did not run on a host with label3", + container.getAllocatedNode().equals(node3.getNodeID())); + } + AppAttemptRemovedSchedulerEvent appRemovedEvent3 = + new AppAttemptRemovedSchedulerEvent( + appAttId3, RMAppAttemptState.FINISHED, false); + + scheduler.handle(appRemovedEvent3); + scheduler.update(); + } +} diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFSParentQueue.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFSParentQueue.java index 671b94be9ed..d37b71e2247 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFSParentQueue.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFSParentQueue.java @@ -25,6 +25,7 @@ import java.util.HashSet; import java.util.Set; +import org.apache.hadoop.yarn.server.resourcemanager.nodelabels.RMNodeLabelsManager; import static org.junit.Assert.assertEquals; import static org.mockito.Mockito.mock; @@ -41,13 +42,17 @@ public void setUp() throws Exception { conf = new FairSchedulerConfiguration(); FairScheduler scheduler = mock(FairScheduler.class); AllocationConfiguration allocConf = new AllocationConfiguration(conf); + RMNodeLabelsManager labelsMgr = new RMNodeLabelsManager(); + + labelsMgr.init(conf); when(scheduler.getAllocationConfiguration()).thenReturn(allocConf); when(scheduler.getConf()).thenReturn(conf); when(scheduler.getResourceCalculator()).thenReturn( new DefaultResourceCalculator()); + when(scheduler.getLabelsManager()).thenReturn(labelsMgr); SystemClock clock = SystemClock.getInstance(); when(scheduler.getClock()).thenReturn(clock); - notEmptyQueues = new HashSet(); + notEmptyQueues = new HashSet<>(); queueManager = new QueueManager(scheduler) { @Override public boolean isEmpty(FSQueue queue) { diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFairScheduler.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFairScheduler.java index cd6a47f4d86..a96cf662131 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFairScheduler.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFairScheduler.java @@ -58,6 +58,8 @@ import org.apache.hadoop.yarn.api.records.ContainerId; import org.apache.hadoop.yarn.api.records.ContainerLaunchContext; import org.apache.hadoop.yarn.api.records.FinalApplicationStatus; +import org.apache.hadoop.yarn.api.records.NodeId; +import org.apache.hadoop.yarn.api.records.NodeLabel; import org.apache.hadoop.yarn.api.records.NodeState; import org.apache.hadoop.yarn.api.records.QueueInfo; import org.apache.hadoop.yarn.api.records.Resource; @@ -94,8 +96,6 @@ import org.apache.hadoop.yarn.server.resourcemanager.rmnode.RMNode; import org.apache.hadoop.yarn.server.resourcemanager.rmnode.RMNodeResourceUpdateEvent; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.AbstractYarnScheduler; - - import org.apache.hadoop.yarn.server.resourcemanager.scheduler.QueueMetrics; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.SchedulerApplicationAttempt; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.SchedulerNode; @@ -133,7 +133,7 @@ public void setUp() throws IOException { scheduler = new FairScheduler(); conf = createConfiguration(); - resourceManager = new MockRM(conf); + resourceManager = new MockRM(conf, null, false, false); ((AsyncDispatcher)resourceManager.getRMContext().getDispatcher()).start(); resourceManager.getRMContext().getStateStore().start(); @@ -157,6 +157,9 @@ public void tearDown() { QueueMetrics.clearQueueMetrics(); DefaultMetricsSystem.shutdown(); YarnAuthorizationProvider.destroy(); + + // Always delete the allocation file when we're done + new File(ALLOC_FILE).delete(); } @@ -266,7 +269,7 @@ public void testMinZeroResourcesSettings() throws IOException { } @Test - public void testAggregateCapacityTracking() throws Exception { + public void testAggregateCapacityTracking() throws IOException { scheduler.init(conf); scheduler.start(); scheduler.reinitialize(conf, resourceManager.getRMContext()); @@ -316,14 +319,19 @@ public void testSimpleFairShareCalculation() throws IOException { scheduler.getQueueManager().getRootQueue().recomputeSteadyShares(); Collection queues = scheduler.getQueueManager().getLeafQueues(); - assertEquals(3, queues.size()); + assertEquals("Unexpected number of queues", 3, queues.size()); // Divided three ways - between the two queues and the default queue for (FSLeafQueue p : queues) { - assertEquals(3414, p.getFairShare().getMemorySize()); - assertEquals(3414, p.getMetrics().getFairShareMB()); - assertEquals(3414, p.getSteadyFairShare().getMemorySize()); - assertEquals(3414, p.getMetrics().getSteadyFairShareMB()); + assertEquals("Fair share for " + p.getName() + " should be 1/3 of the " + + "cluster", 3414, p.getFairShare().getMemorySize()); + assertEquals("Fair share in metrics for " + p.getName() + " should be " + + "1/3 of the cluster", 3414, p.getMetrics().getFairShareMB()); + assertEquals("Steady fair share for " + p.getName() + " should be 1/3 of " + + "the cluster", 3414, p.getSteadyFairShare().getMemorySize()); + assertEquals("Steady fair share in metrics for " + p.getName() + + " should be 1/3 of the cluster", + 3414, p.getMetrics().getSteadyFairShareMB()); } } @@ -981,7 +989,7 @@ public void testHierarchicalQueuesSimilarParents() throws IOException { } @Test - public void testSchedulerRootQueueMetrics() throws Exception { + public void testSchedulerRootQueueMetrics() throws IOException { scheduler.init(conf); scheduler.start(); scheduler.reinitialize(conf, resourceManager.getRMContext()); @@ -1073,7 +1081,7 @@ public void testSimpleContainerAllocation() throws IOException { } @Test (timeout = 5000) - public void testSimpleContainerReservation() throws Exception { + public void testSimpleContainerReservation() throws IOException { scheduler.init(conf); scheduler.start(); scheduler.reinitialize(conf, resourceManager.getRMContext()); @@ -1129,7 +1137,7 @@ public void testSimpleContainerReservation() throws Exception { } @Test (timeout = 5000) - public void testOffSwitchAppReservationThreshold() throws Exception { + public void testOffSwitchAppReservationThreshold() throws IOException { conf.setFloat(FairSchedulerConfiguration.RESERVABLE_NODES, 0.50f); scheduler.init(conf); scheduler.start(); @@ -4407,7 +4415,7 @@ public void testMaxRunningAppsHierarchicalQueues() throws Exception { } @Test - public void testSchedulingOnRemovedNode() throws Exception { + public void testSchedulingOnRemovedNode() { // Disable continuous scheduling, will invoke continuous scheduling manually scheduler.init(conf); scheduler.start(); @@ -4482,7 +4490,7 @@ public void testDefaultRuleInitializesProperlyWhenPolicyNotConfigured() } @Test - public void testBlacklistNodes() throws Exception { + public void testBlacklistNodes() throws IOException { scheduler.init(conf); scheduler.start(); scheduler.reinitialize(conf, resourceManager.getRMContext()); @@ -4537,7 +4545,7 @@ public void testBlacklistNodes() throws Exception { } @Test - public void testGetAppsInQueue() throws Exception { + public void testGetAppsInQueue() throws IOException { scheduler.init(conf); scheduler.start(); scheduler.reinitialize(conf, resourceManager.getRMContext()); @@ -4662,6 +4670,53 @@ public void testMoveWouldViolateMaxResourcesConstraints() throws Exception { } @Test (expected = YarnException.class) + public void testMoveWouldViolateNodeLabelsConstraints() throws Exception { + scheduler.init(conf); + scheduler.start(); + scheduler.reinitialize(conf, resourceManager.getRMContext()); + + QueueManager queueMgr = scheduler.getQueueManager(); + FSLeafQueue oldQueue = queueMgr.getLeafQueue("queue1", true); + FSQueue queue2 = queueMgr.getLeafQueue("queue2", true); + queue2.setMaxShare(new ConfigurableResource(Resource.newInstance(1024, 1))); + + // Add a node + RMNode node = MockNodes.newNodeInfo(1, Resources.createResource(2048, 2)); + + scheduler.handle(new NodeAddedSchedulerEvent(node)); + + // Create two labels + Set clusterLabel = new HashSet<>(); + + clusterLabel.add(NodeLabel.newInstance("label1")); + clusterLabel.add(NodeLabel.newInstance("label2")); + scheduler.getLabelsManager().addToCluserNodeLabels(clusterLabel); + + // Add the first label to the node + Map> nodeLabelMap = new HashMap<>(); + + nodeLabelMap.put(node.getNodeID(), Collections.singleton("label1")); + scheduler.getLabelsManager().addLabelsToNode(nodeLabelMap); + + ApplicationAttemptId appAttId = + createSchedulingRequest(1024, 1, "queue1", "user1", 3, 1, "label1"); + + // Fire two node updates since the default is to only schedule one container + // per heartbeat + NodeUpdateSchedulerEvent updateEvent = new NodeUpdateSchedulerEvent(node); + + scheduler.handle(updateEvent); + scheduler.handle(updateEvent); + + assertEquals("The application did not receive the expected resource " + + "allocation", Resource.newInstance(2048, 2), + oldQueue.getResourceUsage()); + + // Now try moving the queue, which should throw an exception + scheduler.moveApplication(appAttId.getApplicationId(), "queue2"); + } + + @Test (expected = YarnException.class) public void testMoveToNonexistentQueue() throws Exception { scheduler.init(conf); scheduler.start(); @@ -4675,7 +4730,7 @@ public void testMoveToNonexistentQueue() throws Exception { } @Test - public void testLowestCommonAncestorForNonRootParent() throws Exception { + public void testLowestCommonAncestorForNonRootParent() throws IOException { scheduler.init(conf); scheduler.start(); scheduler.reinitialize(conf, resourceManager.getRMContext()); @@ -4696,7 +4751,7 @@ public void testLowestCommonAncestorForNonRootParent() throws Exception { } @Test - public void testLowestCommonAncestorRootParent() throws Exception { + public void testLowestCommonAncestorRootParent() throws IOException { scheduler.init(conf); scheduler.start(); scheduler.reinitialize(conf, resourceManager.getRMContext()); @@ -4717,7 +4772,7 @@ public void testLowestCommonAncestorRootParent() throws Exception { } @Test - public void testLowestCommonAncestorDeeperHierarchy() throws Exception { + public void testLowestCommonAncestorDeeperHierarchy() throws IOException { scheduler.init(conf); scheduler.start(); scheduler.reinitialize(conf, resourceManager.getRMContext()); @@ -4742,7 +4797,7 @@ public void testLowestCommonAncestorDeeperHierarchy() throws Exception { } @Test - public void testDoubleRemoval() throws Exception { + public void testDoubleRemoval() throws IOException { String testUser = "user1"; // convenience var scheduler.init(conf); scheduler.start(); @@ -4841,7 +4896,7 @@ public void testPerfMetricsInited() { } @Test - public void testQueueNameWithTrailingSpace() throws Exception { + public void testQueueNameWithTrailingSpace() throws IOException { scheduler.init(conf); scheduler.start(); scheduler.reinitialize(conf, resourceManager.getRMContext()); @@ -4931,7 +4986,7 @@ public void testEmptyQueueNameInConfigFile() throws IOException { @Test public void testUserAsDefaultQueueWithLeadingTrailingSpaceUserName() - throws Exception { + throws IOException { conf.set(FairSchedulerConfiguration.USER_AS_DEFAULT_QUEUE, "true"); scheduler.init(conf); scheduler.start(); @@ -5051,7 +5106,7 @@ public void handle(Event event) { } @Test(timeout = 120000) - public void testContainerAllocationWithContainerIdLeap() throws Exception { + public void testContainerAllocationWithContainerIdLeap() throws IOException { conf.setFloat(FairSchedulerConfiguration.RESERVABLE_NODES, 0.50f); scheduler.init(conf); scheduler.start(); @@ -5134,14 +5189,20 @@ public void testRefreshQueuesWhenRMHA() throws Exception { new HAServiceProtocol.StateChangeRequestInfo( HAServiceProtocol.RequestSource.REQUEST_BY_USER); - // 1. start a standby RM, file 'ALLOC_FILE' is empty, so there is no queues + // 1. start a standby RM with no queues + PrintWriter out = new PrintWriter(new FileWriter(ALLOC_FILE)); + out.println(""); + out.println(""); + out.println(""); + out.close(); + MockRM rm1 = new MockRM(conf, null); rm1.init(conf); rm1.start(); rm1.getAdminService().transitionToStandby(requestInfo); // 2. add a new queue "test_queue" - PrintWriter out = new PrintWriter(new FileWriter(ALLOC_FILE)); + out = new PrintWriter(new FileWriter(ALLOC_FILE)); out.println(""); out.println(""); out.println(""); diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFairSchedulerEventLog.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFairSchedulerEventLog.java index 09c2370395a..f652d62acd3 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFairSchedulerEventLog.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFairSchedulerEventLog.java @@ -27,7 +27,9 @@ import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.yarn.conf.YarnConfiguration; import org.apache.hadoop.yarn.event.AsyncDispatcher; +import org.apache.hadoop.yarn.server.resourcemanager.RMContextImpl; import org.apache.hadoop.yarn.server.resourcemanager.ResourceManager; +import org.apache.hadoop.yarn.server.resourcemanager.nodelabels.RMNodeLabelsManager; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.ResourceScheduler; import org.junit.After; import org.junit.Before; @@ -41,6 +43,13 @@ @Before public void setUp() throws IOException { scheduler = new FairScheduler(); + + RMContextImpl context = new RMContextImpl(); + RMNodeLabelsManager labelsMgr = new RMNodeLabelsManager(); + + labelsMgr.init(new Configuration()); + context.setNodeLabelManager(labelsMgr); + scheduler.setRMContext(context); Configuration conf = new YarnConfiguration(); conf.setClass(YarnConfiguration.RM_SCHEDULER, FairScheduler.class, diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestMaxRunningAppsEnforcer.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestMaxRunningAppsEnforcer.java index 964ecf560d4..39eb6e341ac 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestMaxRunningAppsEnforcer.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestMaxRunningAppsEnforcer.java @@ -31,6 +31,7 @@ import org.apache.hadoop.yarn.server.resourcemanager.RMContext; import org.apache.hadoop.yarn.api.records.ApplicationAttemptId; import org.apache.hadoop.yarn.api.records.ApplicationId; +import org.apache.hadoop.yarn.server.resourcemanager.nodelabels.RMNodeLabelsManager; import org.apache.hadoop.yarn.util.ControlledClock; import org.apache.hadoop.yarn.util.resource.DefaultResourceCalculator; import org.junit.Before; @@ -48,17 +49,21 @@ @Before public void setup() throws Exception { Configuration conf = new Configuration(); + AllocationConfiguration allocConf = new AllocationConfiguration( + conf); + RMNodeLabelsManager lablesMgr = new RMNodeLabelsManager(); + clock = new ControlledClock(); scheduler = mock(FairScheduler.class); when(scheduler.getConf()).thenReturn( new FairSchedulerConfiguration(conf)); when(scheduler.getClock()).thenReturn(clock); - AllocationConfiguration allocConf = new AllocationConfiguration( - conf); + when(scheduler.getLabelsManager()).thenReturn(lablesMgr); when(scheduler.getAllocationConfiguration()).thenReturn(allocConf); when(scheduler.getResourceCalculator()).thenReturn( new DefaultResourceCalculator()); + lablesMgr.init(conf); queueManager = new QueueManager(scheduler); queueManager.initialize(conf); userMaxApps = allocConf.userMaxApps; diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestQueueManager.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestQueueManager.java index eb2d402cd25..5aa288f6f8e 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestQueueManager.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestQueueManager.java @@ -20,9 +20,11 @@ import static org.junit.Assert.*; import static org.mockito.Mockito.*; +import java.util.Arrays; import java.util.HashSet; import java.util.Set; +import org.apache.hadoop.yarn.server.resourcemanager.nodelabels.RMNodeLabelsManager; import org.apache.hadoop.yarn.util.SystemClock; import org.apache.hadoop.yarn.util.resource.DefaultResourceCalculator; import org.apache.hadoop.yarn.util.resource.Resources; @@ -43,12 +45,22 @@ public void setUp() throws Exception { scheduler = mock(FairScheduler.class); AllocationConfiguration allocConf = new AllocationConfiguration(conf); + RMNodeLabelsManager labelsMgr = new RMNodeLabelsManager(); + + labelsMgr.init(conf); // Set up some queues to test default child max resource inheritance allocConf.configuredQueues.get(FSQueueType.PARENT).add("root.test"); allocConf.configuredQueues.get(FSQueueType.LEAF).add("root.test.childA"); allocConf.configuredQueues.get(FSQueueType.PARENT).add("root.test.childB"); - + allocConf.accessibleNodeLabels.put("root.test", + new HashSet<>(Arrays.asList("label1", "label2"))); + allocConf.accessibleNodeLabels.put("root.test.childA", + new HashSet<>(Arrays.asList("label1"))); + allocConf.accessibleNodeLabels.put("root.test.childB", + new HashSet<>(Arrays.asList(RMNodeLabelsManager.NO_LABEL))); + + when(scheduler.getLabelsManager()).thenReturn(labelsMgr); when(scheduler.getAllocationConfiguration()).thenReturn(allocConf); when(scheduler.getConf()).thenReturn(conf); when(scheduler.getResourceCalculator()).thenReturn( @@ -247,11 +259,72 @@ public void testCreateQueueWithChildDefaults() { } /** + * Test creation of leaf and parent child queues when the parent queue has + * child defaults set. In this test we rely on the root.test, + * root.test.childA and root.test.childB queues that are created in the + * {@link #setUp()} method. + */ + @Test + public void testNodeLabels() { + AllocationConfiguration allocConf = scheduler.getAllocationConfiguration(); + + queueManager.updateAllocationConfiguration(allocConf); + + FSQueue root = queueManager.getParentQueue("root", false); + FSQueue test = queueManager.getParentQueue("root.test", false); + FSQueue childA = queueManager.getLeafQueue("root.test.childA", false); + FSQueue childB = queueManager.getParentQueue("root.test.childB", false); + + assertNotNull("Parent queue root was not created during setup", root); + assertNotNull("Parent queue root.test was not created during setup", test); + assertNotNull("Leaf queue root.test.childA was not created during setup", + childA); + assertNotNull("Parent queue root.test.childB was not created during setup", + childB); + assertEquals("Parent queue root doesn't have the expected node labels", + new HashSet<>(Arrays.asList(RMNodeLabelsManager.ANY)), + root.accessibleLabels); + assertEquals("Parent queue root.test doesn't have the expected node labels", + new HashSet<>(Arrays.asList("label1", "label2")), + test.accessibleLabels); + assertEquals("Leaf queue root.test.childA doesn't have the expected node " + + "labels", new HashSet<>(Arrays.asList("label1")), + childA.accessibleLabels); + assertEquals("Parent queue root.test.childB doesn't have the expected node " + + "labels", new HashSet<>(Arrays.asList(RMNodeLabelsManager.NO_LABEL)), + childB.accessibleLabels); + + FSQueue childC = + queueManager.createQueue("root.test.childC", FSQueueType.LEAF); + assertNotNull("Leaf queue root.test.childC was not created", + queueManager.getLeafQueue("root.test.childC", false)); + assertEquals("Leaf queue root.test.childC doesn't have the expected node " + + "labels", new HashSet<>(Arrays.asList("label1", "label2")), + childC.accessibleLabels); + + FSQueue childD = + queueManager.createQueue("root.test.childB.childD", FSQueueType.LEAF); + assertNotNull("Leaf queue root.test.childB.childD was not created", + queueManager.getLeafQueue("root.test.childC", false)); + assertEquals("Leaf queue root.test.childB.childD doesn't have the expected " + + "node labels", + new HashSet<>(Arrays.asList(RMNodeLabelsManager.NO_LABEL)), + childD.accessibleLabels); + + FSQueue childE = + queueManager.createQueue("root.childE", FSQueueType.LEAF); + assertNotNull("Leaf queue root.childE was not created", + queueManager.getLeafQueue("root.childE", false)); + assertEquals("Leaf queue root.childE doesn't have the expected node labels", + new HashSet<>(Arrays.asList(RMNodeLabelsManager.ANY)), + childE.accessibleLabels); + } + + /** * Test creation of a leaf queue with no resource limits. */ @Test public void testCreateLeafQueueWithDefaults() { - AllocationConfiguration allocConf = scheduler.getAllocationConfiguration(); FSQueue q1 = queueManager.createQueue("root.queue1", FSQueueType.LEAF); assertNotNull("Leaf queue root.queue1 was not created", diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestSchedulingPolicy.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestSchedulingPolicy.java index 3a16454c10a..7e3c8602148 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestSchedulingPolicy.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestSchedulingPolicy.java @@ -31,6 +31,9 @@ import org.apache.hadoop.yarn.api.records.Priority; import org.apache.hadoop.yarn.api.records.Resource; import org.apache.hadoop.yarn.server.resourcemanager.resource.ResourceWeights; +import org.apache.hadoop.yarn.server.resourcemanager.RMContext; +import org.apache.hadoop.yarn.server.resourcemanager.RMContextImpl; +import org.apache.hadoop.yarn.server.resourcemanager.nodelabels.RMNodeLabelsManager; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.policies.DominantResourceFairnessPolicy; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.policies.FairSharePolicy; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.policies.FifoPolicy; @@ -53,6 +56,13 @@ public void setUp() throws Exception { scheduler = new FairScheduler(); conf = new FairSchedulerConfiguration(); + + RMContext context = new RMContextImpl(); + RMNodeLabelsManager labelsMgr = new RMNodeLabelsManager(); + + context.setNodeLabelManager(labelsMgr); + scheduler.setRMContext(context); + labelsMgr.init(conf); } public void testParseSchedulingPolicy() diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebAppFairScheduler.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebAppFairScheduler.java index 8c00b39c4ba..3f562f8989a 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebAppFairScheduler.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebAppFairScheduler.java @@ -23,6 +23,7 @@ import com.google.inject.Injector; import com.google.inject.Module; +import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.util.StringUtils; import org.apache.hadoop.yarn.api.records.ApplicationAttemptId; import org.apache.hadoop.yarn.api.records.ApplicationId; @@ -32,6 +33,8 @@ import org.apache.hadoop.yarn.server.resourcemanager.RMContext; import org.apache.hadoop.yarn.server.resourcemanager.RMContextImpl; import org.apache.hadoop.yarn.server.resourcemanager.ResourceManager; +import org.apache.hadoop.yarn.server.resourcemanager.nodelabels.NullRMNodeLabelsManager; +import org.apache.hadoop.yarn.server.resourcemanager.nodelabels.RMNodeLabelsManager; import org.apache.hadoop.yarn.server.resourcemanager.rmapp.MockRMApp; import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMApp; import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMAppMetrics; @@ -161,6 +164,11 @@ public ResourceScheduler getScheduler() { return mock(AbstractYarnScheduler.class); } }; + + RMNodeLabelsManager nlm = new NullRMNodeLabelsManager(); + + nlm.init(new Configuration()); + rmContext.setNodeLabelManager(nlm); return rmContext; } @@ -178,10 +186,15 @@ private static ResourceManager mockRm(RMContext rmContext) throws private static FairScheduler mockFairScheduler() throws IOException { FairScheduler fs = new FairScheduler(); FairSchedulerConfiguration conf = new FairSchedulerConfiguration(); - fs.setRMContext(new RMContextImpl(null, null, null, null, null, + RMContext rmContext = new RMContextImpl(null, null, null, null, null, null, new RMContainerTokenSecretManager(conf), new NMTokenSecretManagerInRM(conf), - new ClientToAMTokenSecretManagerInRM(), null)); + new ClientToAMTokenSecretManagerInRM(), null); + RMNodeLabelsManager nlm = new RMNodeLabelsManager(); + + nlm.init(conf); + rmContext.setNodeLabelManager(nlm); + fs.setRMContext(rmContext); fs.init(conf); return fs; } diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/TestFairSchedulerQueueInfo.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/TestFairSchedulerQueueInfo.java index 83b7e41112f..7210d5876ea 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/TestFairSchedulerQueueInfo.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/TestFairSchedulerQueueInfo.java @@ -19,6 +19,7 @@ package org.apache.hadoop.yarn.server.resourcemanager.webapp.dao; import org.apache.hadoop.yarn.api.records.Resource; +import org.apache.hadoop.yarn.server.resourcemanager.nodelabels.RMNodeLabelsManager; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.AllocationConfiguration; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.FSQueue; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.FairScheduler; @@ -40,7 +41,10 @@ public void testEmptyChildQueues() throws Exception { FairSchedulerConfiguration conf = new FairSchedulerConfiguration(); FairScheduler scheduler = mock(FairScheduler.class); + RMNodeLabelsManager nlm = new RMNodeLabelsManager(); AllocationConfiguration allocConf = new AllocationConfiguration(conf); + nlm.init(conf); + when(scheduler.getLabelsManager()).thenReturn(nlm); when(scheduler.getAllocationConfiguration()).thenReturn(allocConf); when(scheduler.getConf()).thenReturn(conf); when(scheduler.getClusterResource()).thenReturn(Resource.newInstance(1, 1)); diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/FairScheduler.md hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/FairScheduler.md index e59f86bb51a..9a7710f963d 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/FairScheduler.md +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/FairScheduler.md @@ -109,7 +109,11 @@ The allocation file must be in XML format. The format contains five types of ele * **allowPreemptionFrom**: determines whether the scheduler is allowed to preempt resources from the queue. The default is true. If a queue has this property set to false, this property will apply recursively to all child queues. +<<<<<<< ours * **reservation**: indicates to the `ReservationSystem` that the queue's resources is available for users to reserve. This only applies for leaf queues. A leaf queue is not reservable if this property isn't configured. +======= + * **nodeLabels**: a set of strings that defines the node labels for the queue. If node labels are enabled, a queue will only schedule containers on nodes with matching labels. A queue can have zero or more labels. A node is allowed to have at most a single label. A node matches a queue if the node's label is a member of the queue's set of labels. A queue's label set must be a strict subset of its parent's label set. If a queue has no labels configured, it will inherit its labels from its nearest ancestor that has labels configured. There are two special values for node labels: "*" and "-". If a queue's label set contains "*", then it will match with any node, regardless of the node's label or lack thereof. If the root queue has no label set configured, the root queue's label defaults to "*". If a queue's label set contains only "-", then the queue will only match nodes that have no label. All queues will match a node that has no label, regardless of the queues' label sets or lack thereof. +>>>>>>> theirs * **User elements**: which represent settings governing the behavior of individual users. They can contain a single property: maxRunningApps, a limit on the number of running apps for a particular user. @@ -166,6 +170,7 @@ The allocation file must be in XML format. The format contains five types of ele charlie 5000 mb,0vcores + highmem,gpu diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/NodeLabel.md hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/NodeLabel.md index 39d8311fa08..9115257bdcd 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/NodeLabel.md +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/NodeLabel.md @@ -110,7 +110,7 @@ yarn.resourcemanager.node-labels.provider.fetch-interval-ms | When *"yarn.node-l ###Configuration of Schedulers for node labels -* Capacity Scheduler Configuration +#### Capacity Scheduler Configuration Property | Value ----- | ------ @@ -160,6 +160,25 @@ Notes: * After finishing configuration of CapacityScheduler, execute ```yarn rmadmin -refreshQueues``` to apply changes * Go to scheduler page of RM Web UI to check if you have successfully set configuration. +#### Fair Scheduler Configuration + +The fair scheduler supports exclusive node labels. If a job requests a label, it +will only be run on a node that has that label. If a job requests no label, it +will only be run on a node that has no label. Fair scheduler queues can each +have several labels and will only offer access to nodes with matching labels. +Nodes with no label are available from all queues. + +To configure node labels for the fair scheduler, add a ```nodeLabels``` element +to each queue that should have labels. Any queue without labels specified will +inherit the labels of its parent. If the root queue has no labels specified it +will default to allowing all labels. See the +[fair scheduler documentation](FairScheduler.html) for more information about +the ```nodeLabels``` property and how to set it. + +Fair scheduler queues do not support weights per node label. Instead, relative +weights among node labels can be achieved by creating subqueues with the +desired weights and more narrow node label assignments. + Specifying node label for application -------------------------------------