diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/CapacitySchedulerInfo.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/CapacitySchedulerInfo.java index 548cfe807be..5d143f616e0 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/CapacitySchedulerInfo.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/CapacitySchedulerInfo.java @@ -58,6 +58,8 @@ protected String orderingPolicyInfo; protected String mode; protected String queueType; + protected String creationMethod; + protected String autoCreationEligibility; @XmlTransient static final float EPSILON = 1e-8f; @@ -107,6 +109,9 @@ public CapacitySchedulerInfo(CSQueue parent, CapacityScheduler cs) { } mode = CapacitySchedulerInfoHelper.getMode(parent); queueType = CapacitySchedulerInfoHelper.getQueueType(parent); + creationMethod = CapacitySchedulerInfoHelper.getCreationMethod(parent); + autoCreationEligibility = CapacitySchedulerInfoHelper + .getAutoCreationEligibility(parent, conf); } public float getCapacity() { diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/CapacitySchedulerQueueInfo.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/CapacitySchedulerQueueInfo.java index dec666bf949..60527949d1d 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/CapacitySchedulerQueueInfo.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/CapacitySchedulerQueueInfo.java @@ -91,6 +91,8 @@ protected LeafQueueTemplateInfo leafQueueTemplate; protected String mode; protected String queueType; + protected String creationMethod; + protected String autoCreationEligibility; CapacitySchedulerQueueInfo() { }; @@ -135,8 +137,12 @@ QueueResourceQuotas qResQuotas = q.getQueueResourceQuotas(); populateQueueCapacities(qCapacities, qResQuotas); + CapacitySchedulerConfiguration conf = cs.getConfiguration(); mode = CapacitySchedulerInfoHelper.getMode(q); queueType = CapacitySchedulerInfoHelper.getQueueType(q); + creationMethod = CapacitySchedulerInfoHelper.getCreationMethod(q); + autoCreationEligibility = CapacitySchedulerInfoHelper + .getAutoCreationEligibility(q, conf); ResourceUsage queueResourceUsage = q.getQueueResourceUsage(); populateQueueResourceUsage(queueResourceUsage); @@ -147,7 +153,6 @@ q.getQueueResourceQuotas().getEffectiveMaxResource()); maximumAllocation = new ResourceInfo(q.getMaximumAllocation()); - CapacitySchedulerConfiguration conf = cs.getConfiguration(); queueAcls = new QueueAclsInfo(); for (Map.Entry e : conf .getAcls(queuePath).entrySet()) { diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/helper/CapacitySchedulerInfoHelper.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/helper/CapacitySchedulerInfoHelper.java index 52a5a5cf2a3..4889574ebef 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/helper/CapacitySchedulerInfoHelper.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/helper/CapacitySchedulerInfoHelper.java @@ -18,17 +18,23 @@ import org.apache.hadoop.yarn.exceptions.YarnRuntimeException; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.AbstractCSQueue; -import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.AbstractManagedParentQueue; +import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.AutoCreatedLeafQueue; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CSQueue; +import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacitySchedulerConfiguration; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.LeafQueue; +import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.ManagedParentQueue; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.ParentQueue; public class CapacitySchedulerInfoHelper { - private static final String AUTO_CREATED_LEAF = "autoCreatedLeaf"; - private static final String STATIC_LEAF = "staticLeaf"; - private static final String AUTO_CREATED_PARENT = "autoCreatedParent"; - private static final String STATIC_PARENT = "staticParent"; + private static final String PARENT_QUEUE = "parent"; + private static final String LEAF_QUEUE = "leaf"; private static final String UNKNOWN_QUEUE = "unknown"; + private static final String STATIC_QUEUE = "static"; + private static final String LEGACY_DYNAMIC_QUEUE = "dynamicLegacy"; + private static final String FLEXIBLE_DYNAMIC_QUEUE = "dynamicFlexible"; + private static final String AUTO_CREATION_OFF = "off"; + private static final String AUTO_CREATION_LEGACY = "legacy"; + private static final String AUTO_CREATION_FLEXIBLE = "flexible"; private CapacitySchedulerInfoHelper() {} @@ -52,19 +58,37 @@ public static String getMode(CSQueue queue) throws YarnRuntimeException { public static String getQueueType(CSQueue queue) { if (queue instanceof LeafQueue) { - if (((AbstractCSQueue)queue).isDynamicQueue()) { - return AUTO_CREATED_LEAF; - } else { - return STATIC_LEAF; - } + return LEAF_QUEUE; } else if (queue instanceof ParentQueue) { - if (((AbstractCSQueue)queue).isDynamicQueue()) { - return AUTO_CREATED_PARENT; - } else { - //A ParentQueue with isDynamic=false or an AbstractManagedParentQueue - return STATIC_PARENT; - } + return PARENT_QUEUE; } return UNKNOWN_QUEUE; } + + public static String getCreationMethod(CSQueue queue) { + if (queue instanceof AutoCreatedLeafQueue) { + return LEGACY_DYNAMIC_QUEUE; + } else if (((AbstractCSQueue)queue).isDynamicQueue()) { + return FLEXIBLE_DYNAMIC_QUEUE; + } else { + return STATIC_QUEUE; + } + } + + public static String getAutoCreationEligibility(CSQueue queue, + CapacitySchedulerConfiguration conf) { + if (queue instanceof ManagedParentQueue) { + return AUTO_CREATION_LEGACY; + } else if (queue instanceof ParentQueue && + ((AbstractCSQueue)queue).isDynamicQueue()) { + // A dynamic parent implicitly results in flexible auto creation + // eligibility + return AUTO_CREATION_FLEXIBLE; + } else if (queue instanceof ParentQueue && + conf.isAutoQueueCreationV2Enabled(queue.getQueuePath())) { + return AUTO_CREATION_FLEXIBLE; + } else { + return AUTO_CREATION_OFF; + } + } } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesCapacitySched.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesCapacitySched.java index 61b4df726b0..9ad75d63c98 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesCapacitySched.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesCapacitySched.java @@ -362,7 +362,7 @@ private void verifyClusterScheduler(JSONObject json) throws JSONException, JSONObject info = json.getJSONObject("scheduler"); assertEquals("incorrect number of elements in: " + info, 1, info.length()); info = info.getJSONObject("schedulerInfo"); - assertEquals("incorrect number of elements in: " + info, 16, info.length()); + assertEquals("incorrect number of elements in: " + info, 18, info.length()); verifyClusterSchedulerGeneric(info.getString("type"), (float) info.getDouble("usedCapacity"), (float) info.getDouble("capacity"), @@ -413,10 +413,10 @@ private void verifyClusterSchedulerGeneric(String type, float usedCapacity, private void verifySubQueue(JSONObject info, String q, float parentAbsCapacity, float parentAbsMaxCapacity) throws JSONException, Exception { - int numExpectedElements = 31; + int numExpectedElements = 33; boolean isParentQueue = true; if (!info.has("queues")) { - numExpectedElements = 49; + numExpectedElements = 51; isParentQueue = false; } assertEquals("incorrect number of elements", numExpectedElements, info.length()); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesCapacitySchedDynamicConfig.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesCapacitySchedDynamicConfig.java index 118a0df6271..078dd9ee9c8 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesCapacitySchedDynamicConfig.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesCapacitySchedDynamicConfig.java @@ -34,12 +34,18 @@ import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.http.JettyUtils; +import org.apache.hadoop.yarn.api.records.ApplicationAttemptId; +import org.apache.hadoop.yarn.api.records.ApplicationId; import org.apache.hadoop.yarn.conf.YarnConfiguration; import org.apache.hadoop.yarn.exceptions.YarnException; import org.apache.hadoop.yarn.server.resourcemanager.MockNM; import org.apache.hadoop.yarn.server.resourcemanager.MockRM; +import org.apache.hadoop.yarn.server.resourcemanager.MockRMAppSubmissionData; +import org.apache.hadoop.yarn.server.resourcemanager.MockRMAppSubmitter; import org.apache.hadoop.yarn.server.resourcemanager.ResourceManager; +import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMApp; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.ResourceScheduler; +import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CSQueue; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CSQueueUtils; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacityScheduler; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacitySchedulerAutoQueueHandler; @@ -73,11 +79,14 @@ private static final float EXP_ROOT_WEIGHT_IN_WEIGHT_MODE = 1.0F; private static final float EXP_DEFAULT_WEIGHT_IN_WEIGHT_MODE = 1.0F; private static final double DELTA = 0.00001; - private static final String STATIC_PARENT = "staticParent"; - private static final String STATIC_LEAF = "staticLeaf"; + private static final String PARENT_QUEUE = "parent"; + private static final String LEAF_QUEUE = "leaf"; + private static final String STATIC_QUEUE = "static"; + private static final String FLEXIBLE_DYNAMIC_QUEUE = "dynamicFlexible"; + private static final String AUTO_CREATION_OFF = "off"; + private static final String AUTO_CREATION_LEGACY = "legacy"; + private static final String AUTO_CREATION_FLEXIBLE = "flexible"; private static final int GB = 1024; - private static final String AUTO_CREATED_LEAF = "autoCreatedLeaf"; - private static final String AUTO_CREATED_PARENT = "autoCreatedParent"; protected static MockRM RM; private CapacitySchedulerAutoQueueHandler autoQueueHandler; @@ -88,13 +97,18 @@ public final float weight; public final float normalizedWeight; private String queueType; + private String creationMethod; + private String autoCreationEligibility; public ExpectedQueueWithProperties(String path, float weight, - float normalizedWeight, String queueType) { + float normalizedWeight, String queueType, String creationMethod, + String autoCreationEligibility) { this.path = path; this.weight = weight; this.normalizedWeight = normalizedWeight; this.queueType = queueType; + this.creationMethod = creationMethod; + this.autoCreationEligibility = autoCreationEligibility; } } @@ -161,16 +175,41 @@ public void testSchedulerResponsePercentageMode() validateSchedulerInfo(json, "percentage", new ExpectedQueueWithProperties("root", EXP_WEIGHT_NON_WEIGHT_MODE, EXP_NORM_WEIGHT_NON_WEIGHT_MODE, - STATIC_PARENT), + PARENT_QUEUE, STATIC_QUEUE, AUTO_CREATION_OFF), new ExpectedQueueWithProperties("root.default", EXP_WEIGHT_NON_WEIGHT_MODE, EXP_NORM_WEIGHT_NON_WEIGHT_MODE, - STATIC_LEAF), + LEAF_QUEUE, STATIC_QUEUE, AUTO_CREATION_OFF), new ExpectedQueueWithProperties("root.test1", EXP_WEIGHT_NON_WEIGHT_MODE, EXP_NORM_WEIGHT_NON_WEIGHT_MODE, - STATIC_LEAF), + LEAF_QUEUE, STATIC_QUEUE, AUTO_CREATION_OFF), new ExpectedQueueWithProperties("root.test2", EXP_WEIGHT_NON_WEIGHT_MODE, EXP_NORM_WEIGHT_NON_WEIGHT_MODE, - STATIC_LEAF)); + LEAF_QUEUE, STATIC_QUEUE, AUTO_CREATION_OFF)); + } + + @Test + public void testSchedulerResponsePercentageModeLegacyAutoCreation() + throws Exception { + Configuration config = CSConfigGenerator + .createPercentageConfigLegacyAutoCreation(); + config.set(YarnConfiguration.SCHEDULER_CONFIGURATION_STORE_CLASS, + YarnConfiguration.MEMORY_CONFIGURATION_STORE); + + initResourceManager(config); + JSONObject json = sendRequestToSchedulerEndpoint(); + validateSchedulerInfo(json, "percentage", + new ExpectedQueueWithProperties("root", + EXP_WEIGHT_NON_WEIGHT_MODE, EXP_NORM_WEIGHT_NON_WEIGHT_MODE, + PARENT_QUEUE, STATIC_QUEUE, AUTO_CREATION_OFF), + new ExpectedQueueWithProperties("root.default", + EXP_WEIGHT_NON_WEIGHT_MODE, EXP_NORM_WEIGHT_NON_WEIGHT_MODE, + LEAF_QUEUE, STATIC_QUEUE, AUTO_CREATION_OFF), + new ExpectedQueueWithProperties("root.test1", + EXP_WEIGHT_NON_WEIGHT_MODE, EXP_NORM_WEIGHT_NON_WEIGHT_MODE, + LEAF_QUEUE, STATIC_QUEUE, AUTO_CREATION_OFF), + new ExpectedQueueWithProperties("root.managedtest2", + EXP_WEIGHT_NON_WEIGHT_MODE, EXP_NORM_WEIGHT_NON_WEIGHT_MODE, + PARENT_QUEUE, STATIC_QUEUE, AUTO_CREATION_LEGACY)); } @Test @@ -186,16 +225,16 @@ public void testSchedulerResponseAbsoluteMode() validateSchedulerInfo(json, "absolute", new ExpectedQueueWithProperties("root", EXP_WEIGHT_NON_WEIGHT_MODE, EXP_NORM_WEIGHT_NON_WEIGHT_MODE, - STATIC_PARENT), + PARENT_QUEUE, STATIC_QUEUE, AUTO_CREATION_OFF), new ExpectedQueueWithProperties("root.default", EXP_WEIGHT_NON_WEIGHT_MODE, EXP_NORM_WEIGHT_NON_WEIGHT_MODE, - STATIC_LEAF), + LEAF_QUEUE, STATIC_QUEUE, AUTO_CREATION_OFF), new ExpectedQueueWithProperties("root.test1", EXP_WEIGHT_NON_WEIGHT_MODE, EXP_NORM_WEIGHT_NON_WEIGHT_MODE, - STATIC_LEAF), + LEAF_QUEUE, STATIC_QUEUE, AUTO_CREATION_OFF), new ExpectedQueueWithProperties("root.test2", EXP_WEIGHT_NON_WEIGHT_MODE, EXP_NORM_WEIGHT_NON_WEIGHT_MODE, - STATIC_LEAF)); + LEAF_QUEUE, STATIC_QUEUE, AUTO_CREATION_OFF)); } @Test @@ -211,13 +250,13 @@ public void testSchedulerResponseWeightMode() validateSchedulerInfo(json, "weight", new ExpectedQueueWithProperties("root", EXP_ROOT_WEIGHT_IN_WEIGHT_MODE, EXP_ROOT_WEIGHT_IN_WEIGHT_MODE, - STATIC_PARENT), + PARENT_QUEUE, STATIC_QUEUE, AUTO_CREATION_OFF), new ExpectedQueueWithProperties("root.default", 10.0f, 0.5f, - STATIC_LEAF), + LEAF_QUEUE, STATIC_QUEUE, AUTO_CREATION_OFF), new ExpectedQueueWithProperties("root.test1", 4.0f, 0.2f, - STATIC_LEAF), + LEAF_QUEUE, STATIC_QUEUE, AUTO_CREATION_OFF), new ExpectedQueueWithProperties("root.test2", 6.0f, 0.3f, - STATIC_LEAF)); + LEAF_QUEUE, STATIC_QUEUE, AUTO_CREATION_OFF)); } @Test @@ -234,13 +273,13 @@ public void testSchedulerResponseWeightModeWithAutoCreatedQueues() validateSchedulerInfo(json, "weight", new ExpectedQueueWithProperties("root", EXP_ROOT_WEIGHT_IN_WEIGHT_MODE, EXP_ROOT_WEIGHT_IN_WEIGHT_MODE, - STATIC_PARENT), + PARENT_QUEUE, STATIC_QUEUE, AUTO_CREATION_OFF), new ExpectedQueueWithProperties("root.default", 10.0f, 0.5f, - STATIC_LEAF), + LEAF_QUEUE, STATIC_QUEUE, AUTO_CREATION_OFF), new ExpectedQueueWithProperties("root.test1", 4.0f, 0.2f, - STATIC_LEAF), + LEAF_QUEUE, STATIC_QUEUE, AUTO_CREATION_OFF), new ExpectedQueueWithProperties("root.test2", 6.0f, 0.3f, - STATIC_LEAF)); + LEAF_QUEUE, STATIC_QUEUE, AUTO_CREATION_OFF)); //Now create some auto created queues createQueue("root.auto1"); @@ -255,42 +294,42 @@ public void testSchedulerResponseWeightModeWithAutoCreatedQueues() int sumOfWeights = 24; ExpectedQueueWithProperties expectedRootQ = new ExpectedQueueWithProperties("root", - EXP_ROOT_WEIGHT_IN_WEIGHT_MODE, EXP_ROOT_WEIGHT_IN_WEIGHT_MODE, - STATIC_PARENT); + EXP_ROOT_WEIGHT_IN_WEIGHT_MODE, EXP_ROOT_WEIGHT_IN_WEIGHT_MODE, + PARENT_QUEUE, STATIC_QUEUE, AUTO_CREATION_OFF); validateSchedulerInfo(json, "weight", expectedRootQ, new ExpectedQueueWithProperties("root.auto1", EXP_DEFAULT_WEIGHT_IN_WEIGHT_MODE, EXP_DEFAULT_WEIGHT_IN_WEIGHT_MODE / sumOfWeights, - AUTO_CREATED_LEAF), + LEAF_QUEUE, FLEXIBLE_DYNAMIC_QUEUE, AUTO_CREATION_OFF), new ExpectedQueueWithProperties("root.auto2", EXP_DEFAULT_WEIGHT_IN_WEIGHT_MODE, EXP_DEFAULT_WEIGHT_IN_WEIGHT_MODE / sumOfWeights, - AUTO_CREATED_LEAF), + LEAF_QUEUE, FLEXIBLE_DYNAMIC_QUEUE, AUTO_CREATION_OFF), new ExpectedQueueWithProperties("root.auto3", EXP_DEFAULT_WEIGHT_IN_WEIGHT_MODE, EXP_DEFAULT_WEIGHT_IN_WEIGHT_MODE / sumOfWeights, - AUTO_CREATED_LEAF), + LEAF_QUEUE, FLEXIBLE_DYNAMIC_QUEUE, AUTO_CREATION_OFF), new ExpectedQueueWithProperties("root.autoParent1", EXP_DEFAULT_WEIGHT_IN_WEIGHT_MODE, EXP_DEFAULT_WEIGHT_IN_WEIGHT_MODE / sumOfWeights, - AUTO_CREATED_PARENT), + PARENT_QUEUE, FLEXIBLE_DYNAMIC_QUEUE, AUTO_CREATION_FLEXIBLE), new ExpectedQueueWithProperties("root.default", 10.0f, 10.0f / sumOfWeights, - STATIC_LEAF), + LEAF_QUEUE, STATIC_QUEUE, AUTO_CREATION_OFF), new ExpectedQueueWithProperties("root.test1", 4.0f, 4.0f / sumOfWeights, - STATIC_LEAF), + LEAF_QUEUE, STATIC_QUEUE, AUTO_CREATION_OFF), new ExpectedQueueWithProperties("root.test2", 6.0f, 6.0f / sumOfWeights, - STATIC_LEAF)); + LEAF_QUEUE, STATIC_QUEUE, AUTO_CREATION_OFF)); validateChildrenOfParent(json, "root.autoParent1", "weight", expectedRootQ, new ExpectedQueueWithProperties("root.autoParent1.auto4", EXP_DEFAULT_WEIGHT_IN_WEIGHT_MODE, EXP_DEFAULT_WEIGHT_IN_WEIGHT_MODE, - AUTO_CREATED_LEAF)); + LEAF_QUEUE, FLEXIBLE_DYNAMIC_QUEUE, AUTO_CREATION_OFF)); } private void initAutoQueueHandler() throws Exception { @@ -300,6 +339,18 @@ private void initAutoQueueHandler() throws Exception { MockNM nm1 = RM.registerNode("h1:1234", 1200 * GB); // label = x } + private void submitApp(String leafQueueName, String user) throws Exception { + MockRMAppSubmissionData data = + MockRMAppSubmissionData.Builder.createWithMemory(GB, RM) + .withAppName("test-auto-queue-activation") + .withUser(user) + .withAcls(null) + .withQueue(leafQueueName) + .withUnmanagedAM(false) + .build(); + MockRMAppSubmitter.submit(RM, data); + } + private LeafQueue createQueue(String queuePath) throws YarnException { return autoQueueHandler.autoCreateQueue( CSQueueUtils.extractQueuePath(queuePath)); @@ -433,9 +484,18 @@ private void verifyQueues(String parentPath, String expectedMode, Float.parseFloat(obj.getString("normalizedWeight")), DELTA); //validate queue creation type - Assert.assertEquals("Queue creation type does not match for queue " + + Assert.assertEquals("Queue type does not match for queue " + queuePath, expectedQueue.queueType, obj.getString("queueType")); + + Assert.assertEquals("Queue creation type does not match for queue " + + queuePath, + expectedQueue.creationMethod, obj.getString("creationMethod")); + + Assert.assertEquals("Queue auto creation eligibility does not " + + "match for queue " + queuePath, + expectedQueue.autoCreationEligibility, + obj.getString("autoCreationEligibility")); } //Validate queue paths and modes @@ -470,6 +530,20 @@ public static Configuration createPercentageConfig() { return createConfiguration(conf); } + public static Configuration createPercentageConfigLegacyAutoCreation() { + Map conf = new HashMap<>(); + conf.put("yarn.scheduler.capacity.root.queues", "default, test1, " + + "managedtest2"); + conf.put("yarn.scheduler.capacity.root.test1.capacity", "50"); + conf.put("yarn.scheduler.capacity.root.managedtest2.capacity", "50"); + conf.put("yarn.scheduler.capacity.root.test1.maximum-capacity", "100"); + conf.put("yarn.scheduler.capacity.root.test1.state", "RUNNING"); + conf.put("yarn.scheduler.capacity.root.managedtest2.state", "RUNNING"); + conf.put("yarn.scheduler.capacity.root.managedtest2." + + "auto-create-child-queue.enabled", "true"); + return createConfiguration(conf); + } + public static Configuration createAbsoluteConfig() { Map conf = new HashMap<>(); conf.put("yarn.scheduler.capacity.root.queues", "default, test1, test2"); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesForCSWithPartitions.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesForCSWithPartitions.java index 7a28c914016..8d9c453731c 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesForCSWithPartitions.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesForCSWithPartitions.java @@ -574,7 +574,7 @@ private void verifySchedulerInfoJson(JSONObject json) JSONObject info = json.getJSONObject("scheduler"); assertEquals("incorrect number of elements", 1, info.length()); info = info.getJSONObject("schedulerInfo"); - assertEquals("incorrect number of elements", 16, info.length()); + assertEquals("incorrect number of elements", 18, info.length()); JSONObject capacitiesJsonObject = info.getJSONObject(CAPACITIES); JSONArray partitionsCapsArray = capacitiesJsonObject.getJSONArray(QUEUE_CAPACITIES_BY_PARTITION);