diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacitySchedulerConfiguration.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacitySchedulerConfiguration.java index d0ee25df300..322f892f506 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacitySchedulerConfiguration.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacitySchedulerConfiguration.java @@ -2307,16 +2307,44 @@ private void updateMinMaxResourceToConf(String label, String queue, public boolean checkConfigTypeIsAbsoluteResource(String label, String queue, Set resourceTypes) { String propertyName = getNodeLabelPrefix(queue, label) + CAPACITY; + String propertyNameMax = getNodeLabelPrefix(queue, label) + MAXIMUM_CAPACITY; String resourceString = get(propertyName); - if (resourceString == null || resourceString.isEmpty()) { + String resourceStringMax = get(propertyNameMax); + + //Both null. + if ((resourceString == null || resourceString.isEmpty()) + && (resourceStringMax == null || resourceStringMax.isEmpty())) { return false; } - Matcher matcher = RESOURCE_PATTERN.matcher(resourceString); - if (matcher.find()) { + //Check resource configuration + //consistency between capacity and maximum-capacity. + Matcher matcher = null; + Matcher matcherMax = null; + if (resourceString != null) { + matcher = RESOURCE_PATTERN.matcher(resourceString); + } + if (resourceStringMax != null) { + matcherMax = RESOURCE_PATTERN.matcher(resourceStringMax); + } + if ((resourceString == null || resourceString.isEmpty())) { + return matcherMax.find(); + } + if ((resourceStringMax == null || resourceStringMax.isEmpty())) { + return matcher.find(); + } + + //Both not null. + if (matcher.find() && matcherMax.find()) { return true; + } else if (!matcher.find() && !matcherMax.find()) { + return false; + } else { + throw new IllegalArgumentException("Queue '" + queue + + "' should use consistent percentage based capacity" + + " configuration or absolute resource " + + "between capacity and maximum-capacity."); } - return false; } private Resource internalGetLabeledResourceRequirementForQueue(String queue, diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestAbsoluteResourceConfiguration.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestAbsoluteResourceConfiguration.java index 3d5637c3522..8b64b591ebb 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestAbsoluteResourceConfiguration.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestAbsoluteResourceConfiguration.java @@ -22,6 +22,7 @@ import java.util.HashSet; import java.util.Set; +import org.apache.hadoop.test.LambdaTestUtils; import org.apache.hadoop.yarn.api.records.Resource; import org.apache.hadoop.yarn.conf.YarnConfiguration; import org.apache.hadoop.yarn.server.resourcemanager.MockNM; @@ -471,7 +472,13 @@ public void testComplexValidateAbsoluteResourceConfig() throws Exception { // create conf with basic queue configuration. CapacitySchedulerConfiguration csConf = setupComplexQueueConfiguration( false); - setupComplexMinMaxResourceConfig(csConf); + // Update min/max resource to queueA/B/C + csConf.setMinimumResourceRequirement("", QUEUEA_FULL, QUEUE_A_MINRES); + csConf.setMinimumResourceRequirement("", QUEUEB_FULL, QUEUE_B_MINRES); + csConf.setMinimumResourceRequirement("", QUEUEC_FULL, QUEUE_C_MINRES); + csConf.setMinimumResourceRequirement("", QUEUEA1_FULL, QUEUE_A1_MINRES); + csConf.setMinimumResourceRequirement("", QUEUEA2_FULL, QUEUE_A2_MINRES); + csConf.setMinimumResourceRequirement("", QUEUEB1_FULL, QUEUE_B1_MINRES); csConf.setClass(YarnConfiguration.RM_SCHEDULER, CapacityScheduler.class, ResourceScheduler.class); @@ -817,4 +824,45 @@ public void testEffectiveResourceAfterIncreasingClusterResource() rm.stop(); } + + @Test + public void testUnConsistentResourceConfig() throws Exception { + /** + * root + * / | + * a b + * + * Test below cases: 1) Test unconsistent case a,b capacity with + * percentage resource, a with maximum-capacity absolute resource. + */ + + // create conf with basic queue configuration. + CapacitySchedulerConfiguration csConf = + new CapacitySchedulerConfiguration(); + csConf.setQueues(CapacitySchedulerConfiguration.ROOT, + new String[] {QUEUEA, QUEUEB}); + + // Set default capacities with percentage. + csConf.setCapacity(QUEUEA_FULL, 50f); + csConf.setCapacity(QUEUEB_FULL, 50f); + + csConf.setClass(YarnConfiguration.RM_SCHEDULER, CapacityScheduler.class, + ResourceScheduler.class); + + @SuppressWarnings("resource") + MockRM rm = new MockRM(csConf); + rm.start(); + + // Add few nodes + rm.registerNode("127.0.0.1:1234", 250 * GB, 50); + + csConf.setMaximumResourceRequirement("",QUEUEA_FULL, QUEUE_A_MAXRES); + CapacityScheduler cs = (CapacityScheduler) rm.getResourceScheduler(); + LambdaTestUtils.intercept(IOException.class, + "Queue '" + QUEUEA_FULL + + "' should use consistent percentage based capacity" + + " configuration or absolute resource " + + "between capacity and maximum-capacity.", + () -> cs.reinitialize(csConf, rm.getRMContext())); + } }