diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/SchedulerUtils.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/SchedulerUtils.java index a80e921..c066177 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/SchedulerUtils.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/SchedulerUtils.java @@ -22,8 +22,11 @@ import java.util.Set; import org.apache.commons.lang.StringUtils; +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; import org.apache.hadoop.classification.InterfaceAudience.Private; import org.apache.hadoop.classification.InterfaceStability.Unstable; +import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.yarn.api.records.ContainerExitStatus; import org.apache.hadoop.yarn.api.records.ContainerId; import org.apache.hadoop.yarn.api.records.ContainerState; @@ -32,6 +35,7 @@ import org.apache.hadoop.yarn.api.records.QueueInfo; import org.apache.hadoop.yarn.api.records.Resource; import org.apache.hadoop.yarn.api.records.ResourceRequest; +import org.apache.hadoop.yarn.conf.YarnConfiguration; import org.apache.hadoop.yarn.exceptions.InvalidLabelResourceRequestException; import org.apache.hadoop.yarn.exceptions.InvalidResourceRequestException; import org.apache.hadoop.yarn.factories.RecordFactory; @@ -50,6 +54,8 @@ @Unstable public class SchedulerUtils { + private static final Log LOG = LogFactory.getLog(SchedulerUtils.class); + private static final RecordFactory recordFactory = RecordFactoryProvider.getRecordFactory(null); @@ -192,7 +198,7 @@ public static void normalizeRequest( } private static void normalizeNodeLabelExpressionInRequest( - ResourceRequest resReq, QueueInfo queueInfo) { + ResourceRequest resReq, QueueInfo queueInfo, boolean isLabelEnabled) { String labelExp = resReq.getNodeLabelExpression(); @@ -207,7 +213,16 @@ private static void normalizeNodeLabelExpressionInRequest( if (labelExp == null) { labelExp = RMNodeLabelsManager.NO_LABEL; } - resReq.setNodeLabelExpression(labelExp); + // If Node label is not enabled reset request to no label + if (!isLabelEnabled) { + if (LOG.isDebugEnabled()) { + LOG.debug("Resetting label in request to NO_LABEL," + + "Node labels not enabled in cluster"); + } + resReq.setNodeLabelExpression(RMNodeLabelsManager.NO_LABEL); + } else { + resReq.setNodeLabelExpression(labelExp); + } } public static void normalizeAndValidateRequest(ResourceRequest resReq, @@ -222,7 +237,12 @@ public static void normalizeAndValidateRequest(ResourceRequest resReq, Resource maximumResource, String queueName, YarnScheduler scheduler, boolean isRecovery, RMContext rmContext, QueueInfo queueInfo) throws InvalidResourceRequestException { - if (null == queueInfo) { + Configuration conf = rmContext.getYarnConfiguration(); + boolean isLabelEnabled = false; + if (null != conf) { + isLabelEnabled = YarnConfiguration.areNodeLabelsEnabled(conf); + } + if (null == queueInfo && isLabelEnabled) { try { queueInfo = scheduler.getQueueInfo(queueName, false, false); } catch (IOException e) { @@ -230,9 +250,11 @@ public static void normalizeAndValidateRequest(ResourceRequest resReq, // the queueInfo here, and move forward } } - SchedulerUtils.normalizeNodeLabelExpressionInRequest(resReq, queueInfo); + SchedulerUtils.normalizeNodeLabelExpressionInRequest(resReq, queueInfo, + isLabelEnabled); if (!isRecovery) { - validateResourceRequest(resReq, maximumResource, queueInfo, rmContext); + validateResourceRequest(resReq, maximumResource, queueInfo, rmContext, + isLabelEnabled); } } @@ -259,8 +281,8 @@ public static void normalizeAndvalidateRequest(ResourceRequest resReq, * @throws InvalidResourceRequestException when there is invalid request */ private static void validateResourceRequest(ResourceRequest resReq, - Resource maximumResource, QueueInfo queueInfo, RMContext rmContext) - throws InvalidResourceRequestException { + Resource maximumResource, QueueInfo queueInfo, RMContext rmContext, + boolean isLabelEnabled) throws InvalidResourceRequestException { if (resReq.getCapability().getMemory() < 0 || resReq.getCapability().getMemory() > maximumResource.getMemory()) { throw new InvalidResourceRequestException("Invalid resource request" @@ -280,48 +302,54 @@ private static void validateResourceRequest(ResourceRequest resReq, + ", maxVirtualCores=" + maximumResource.getVirtualCores()); } String labelExp = resReq.getNodeLabelExpression(); - // we don't allow specify label expression other than resourceName=ANY now - if (!ResourceRequest.ANY.equals(resReq.getResourceName()) - && labelExp != null && !labelExp.trim().isEmpty()) { - throw new InvalidResourceRequestException( - "Invailid resource request, queue=" + queueInfo.getQueueName() - + " specified node label expression in a " - + "resource request has resource name = " - + resReq.getResourceName()); - } - - // we don't allow specify label expression with more than one node labels now - if (labelExp != null && labelExp.contains("&&")) { - throw new InvalidLabelResourceRequestException( - "Invailid resource request, queue=" + queueInfo.getQueueName() - + " specified more than one node label " - + "in a node label expression, node label expression = " - + labelExp); - } - - if (labelExp != null && !labelExp.trim().isEmpty() && queueInfo != null) { - if (!checkQueueLabelExpression(queueInfo.getAccessibleNodeLabels(), - labelExp, rmContext)) { + if (isLabelEnabled) { + // we don't allow specify label expression other than resourceName=ANY now + if (!ResourceRequest.ANY.equals(resReq.getResourceName()) + && labelExp != null && !labelExp.trim().isEmpty()) { throw new InvalidLabelResourceRequestException( - "Invalid resource request" - + ", queue=" - + queueInfo.getQueueName() - + " doesn't have permission to access all labels " - + "in resource request. labelExpression of resource request=" - + labelExp - + ". Queue labels=" - + (queueInfo.getAccessibleNodeLabels() == null ? "" : StringUtils.join(queueInfo - .getAccessibleNodeLabels().iterator(), ','))); + "Invalid resource request, queue=" + queueInfo.getQueueName() + + " specified node label expression in a " + + "resource request has resource name = " + + resReq.getResourceName()); + } + + // we don't allow specify label expression with more than one node labels + // now + if (labelExp != null && labelExp.contains("&&")) { + throw new InvalidLabelResourceRequestException( + "Invalid resource request, queue=" + queueInfo.getQueueName() + + " specified more than one node label " + + "in a node label expression, node label expression = " + + labelExp); + } + + if (labelExp != null && !labelExp.trim().isEmpty() && queueInfo != null) { + if (!checkQueueLabelExpression(queueInfo.getAccessibleNodeLabels(), + labelExp)) { + throw new InvalidLabelResourceRequestException( + "Invalid resource request" + ", queue=" + queueInfo.getQueueName() + + " doesn't have permission to access all labels " + + "in resource request. labelExpression of resource request=" + + labelExp + ". Queue labels=" + + (queueInfo.getAccessibleNodeLabels() == null ? "" + : StringUtils.join( + queueInfo.getAccessibleNodeLabels().iterator(), + ','))); + } else { + checkQueueLabelInLabelManager(queueInfo.getAccessibleNodeLabels(), + labelExp, rmContext); + } } } } /** * Check queue label expression, check if node label in queue's - * node-label-expression existed in clusterNodeLabels if rmContext != null + * node-label-expression. + * */ public static boolean checkQueueLabelExpression(Set queueLabels, - String labelExpression, RMContext rmContext) { + String labelExpression) { // if label expression is empty, we can allocate container on any node if (labelExpression == null) { return true; @@ -331,19 +359,42 @@ public static boolean checkQueueLabelExpression(Set queueLabels, if (!str.trim().isEmpty()) { // check queue label if (queueLabels == null) { - return false; + return false; } else { if (!queueLabels.contains(str) && !queueLabels.contains(RMNodeLabelsManager.ANY)) { return false; } } - + } + } + return true; + } + + /** + * Check queue label Expression existed in clusterNodeLabels if rmContext != + * null. + * + * @param queueLabels + * @param labelExpression + * @param rmContext + * @return + * @throws InvalidLabelResourceRequestException + */ + private static boolean checkQueueLabelInLabelManager(Set queueLabels, + String labelExpression, RMContext rmContext) + throws InvalidLabelResourceRequestException { + for (String str : labelExpression.split("&&")) { + str = str.trim(); + if (!str.trim().isEmpty()) { // check node label manager contains this label if (null != rmContext) { RMNodeLabelsManager nlm = rmContext.getNodeLabelManager(); if (nlm != null && !nlm.containsNodeLabel(str)) { - return false; + throw new InvalidLabelResourceRequestException( + "Invalid label resource request, cluster do not contain " + + ", label= " + str + " Cluster labels= " + + StringUtils.join(nlm.getClusterNodeLabels(), ',')); } } } @@ -351,7 +402,6 @@ public static boolean checkQueueLabelExpression(Set queueLabels, return true; } - public static AccessType toAccessType(QueueACL acl) { switch (acl) { case ADMINISTER_QUEUE: diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/LeafQueue.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/LeafQueue.java index 9c6d8ee..d9d216c 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/LeafQueue.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/LeafQueue.java @@ -177,18 +177,15 @@ protected synchronized void setupQueueConfigs(Resource clusterResource) maxAMResourcePerQueuePercent = conf.getMaximumApplicationMasterResourcePerQueuePercent(getQueuePath()); - if (!SchedulerUtils.checkQueueLabelExpression( - this.accessibleLabels, this.defaultLabelExpression, null)) { - throw new IOException("Invalid default label expression of " - + " queue=" - + getQueueName() - + " doesn't have permission to access all labels " + if (!SchedulerUtils.checkQueueLabelExpression(this.accessibleLabels, + this.defaultLabelExpression)) { + throw new IOException("Invalid default label expression of " + " queue=" + + getQueueName() + " doesn't have permission to access all labels " + "in default label expression. labelExpression of resource request=" + (this.defaultLabelExpression == null ? "" : this.defaultLabelExpression) - + ". Queue labels=" - + (getAccessibleNodeLabels() == null ? "" : StringUtils.join( - getAccessibleNodeLabels().iterator(), ','))); + + ". Queue labels=" + (getAccessibleNodeLabels() == null ? "" + : StringUtils.join(getAccessibleNodeLabels().iterator(), ','))); } nodeLocalityDelay = conf.getNodeLocalityDelay(); diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/TestSchedulerUtils.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/TestSchedulerUtils.java index 0e84d38..15268d1 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/TestSchedulerUtils.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/TestSchedulerUtils.java @@ -85,6 +85,7 @@ import org.apache.hadoop.yarn.util.resource.ResourceCalculator; import org.apache.hadoop.yarn.util.resource.Resources; import org.junit.Assert; +import org.junit.Before; import org.junit.Test; import com.google.common.collect.ImmutableSet; @@ -95,6 +96,7 @@ private static final Log LOG = LogFactory.getLog(TestSchedulerUtils.class); private RMContext rmContext = getMockRMContext(); + private static YarnConfiguration conf = new YarnConfiguration(); @Test (timeout = 30000) public void testNormalizeRequest() { @@ -258,7 +260,6 @@ public void testValidateResourceRequestWithErrorLabelsPermission() resReq.setNodeLabelExpression("x"); SchedulerUtils.normalizeAndvalidateRequest(resReq, maxResource, "queue", scheduler, rmContext); - fail("Should fail"); } catch (InvalidResourceRequestException e) { } @@ -464,6 +465,34 @@ public void testValidateResourceRequestWithErrorLabelsPermission() rmContext.getNodeLabelManager().removeFromClusterNodeLabels( Arrays.asList("x")); } + + try { + Resource resource = Resources.createResource(0, + YarnConfiguration.DEFAULT_RM_SCHEDULER_MINIMUM_ALLOCATION_VCORES); + ResourceRequest resReq1 = BuilderUtils + .newResourceRequest(mock(Priority.class), "*", resource, 1, "x"); + SchedulerUtils.normalizeAndvalidateRequest(resReq1, maxResource, "queue", + scheduler, rmContext); + fail("Should fail"); + } catch (InvalidResourceRequestException e) { + assertEquals("Invalid label resource request, cluster do not contain , " + + "label= x Cluster labels= ", e.getMessage()); + } + + try { + rmContext.getYarnConfiguration() + .set(YarnConfiguration.NODE_LABELS_ENABLED, "false"); + Resource resource = Resources.createResource(0, + YarnConfiguration.DEFAULT_RM_SCHEDULER_MINIMUM_ALLOCATION_VCORES); + ResourceRequest resReq1 = BuilderUtils + .newResourceRequest(mock(Priority.class), "*", resource, 1, "x"); + SchedulerUtils.normalizeAndvalidateRequest(resReq1, maxResource, "queue", + scheduler, rmContext); + Assert.assertEquals(RMNodeLabelsManager.NO_LABEL, + resReq1.getNodeLabelExpression()); + } catch (InvalidResourceRequestException e) { + fail("Should not get exception"); + } } @Test (timeout = 30000) @@ -774,6 +803,13 @@ private static RMContext getMockRMContext() { RMNodeLabelsManager nlm = new NullRMNodeLabelsManager(); nlm.init(new Configuration(false)); when(rmContext.getNodeLabelManager()).thenReturn(nlm); + when(rmContext.getYarnConfiguration()).thenReturn(conf); return rmContext; } + + @Before + public void before() { + rmContext.getYarnConfiguration().set(YarnConfiguration.NODE_LABELS_ENABLED, + "true"); + } }