diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/monitor/capacity/ProportionalCapacityPreemptionPolicyMockFramework.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/monitor/capacity/ProportionalCapacityPreemptionPolicyMockFramework.java index c0e95ca..248f426 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/monitor/capacity/ProportionalCapacityPreemptionPolicyMockFramework.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/monitor/capacity/ProportionalCapacityPreemptionPolicyMockFramework.java @@ -326,9 +326,13 @@ private void mockApplications(String appsConfig) { HashMap> userMap = new HashMap>(); HashMap>> userResourceUsagePerLabel = new HashMap<>(); LeafQueue queue = null; + int mulp = -1; for (String a : appsConfig.split(";")) { String[] strs = a.split("\t"); String queueName = strs[0]; + if (mulp <= 0 && strs.length > 2 && strs[2] != null) { + mulp = 100 / (new Integer(strs[2]).intValue()); + } // get containers List liveContainers = new ArrayList(); @@ -399,8 +403,13 @@ private void mockApplications(String appsConfig) { Resource capacity = Resources.multiply(totResoucePerPartition, queue.getQueueCapacities().getAbsoluteCapacity()); HashSet users = userMap.get(queue.getQueueName()); - Resource userLimit = Resources.divideAndCeil(rc, capacity, - users.size()); + Resource userLimit; + if (mulp > 0) { + userLimit = Resources.divideAndCeil(rc, capacity, mulp); + } else { + userLimit = Resources.divideAndCeil(rc, capacity, + users.size()); + } LOG.debug("Updating user-limit from mock: totResoucePerPartition=" + totResoucePerPartition + ", capacity=" + capacity + ", users.size()=" + users.size() + ", userlimit= " + userLimit diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/monitor/capacity/TestProportionalCapacityPreemptionPolicyIntraQueueUserLimit.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/monitor/capacity/TestProportionalCapacityPreemptionPolicyIntraQueueUserLimit.java index ce04630..66cbc99 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/monitor/capacity/TestProportionalCapacityPreemptionPolicyIntraQueueUserLimit.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/monitor/capacity/TestProportionalCapacityPreemptionPolicyIntraQueueUserLimit.java @@ -505,4 +505,184 @@ public void testIntraQueuePreemptionWithTwoRequestingUsers() new TestProportionalCapacityPreemptionPolicy.IsPreemptionRequestFor( getAppAttemptId(2)))); } + + @Test + public void testNoIntraQueuePreemptionIfBelowUserLimitAndLowPriorityExtraUsers() + throws IOException { + /** + * Queue structure is: + * + *
+     *       root
+     *        |
+     *        a
+     * 
+ * + * Scenario: + * Preconditions: + * Queue total resources: 100 + * Minimum user limit percent: 50% + * +--------------+----------+------+---------+ + * | APP | USER | PRIORITY | USED | PENDING | + * +--------------+----------+------+---------+ + * | app1 | user1 | 1 | 50 | 0 | + * | app2 | user2 | 1 | 50 | 0 | + * | app3 | user3 | 0 | 0 | 10 | + * +--------------+----------+------+---------+ + * This scenario should never preempt from either user1 or user2 + */ + + // Set max preemption per round to 50% (this is different from minimum user + // limit percent). + conf.setFloat(CapacitySchedulerConfiguration. + INTRAQUEUE_PREEMPTION_MAX_ALLOWABLE_LIMIT, + (float) 0.7); + + String labelsConfig = "=100,true;"; + String nodesConfig = // n1 has no label + "n1= res=100"; + String queuesConfig = + // guaranteed,max,used,pending,reserved + "root(=[100 100 100 10 0]);" + // root + "-a(=[100 100 100 10 0])"; // a + + String appsConfig = + // queueName\t\ + // (priority,resource,host,label,#repeat,reserved,pending,user)\tMULP; + "a\t(1,1,n1,,50,false,0,user1)\t50;" + // app1, user1 + "a\t(1,1,n1,,50,false,0,user2)\t50;" + // app2, user2 + "a\t(0,1,n1,,0,false,10,user3)\t50"; // app3, user3 + + buildEnv(labelsConfig, nodesConfig, queuesConfig, appsConfig); + policy.editSchedule(); + + // app2/app4 needs more resource and its well under its user-limit. Hence + // preempt resources from app1 (compare to app3, app1 has low priority). + verify(mDisp, times(0)).handle(argThat( + new TestProportionalCapacityPreemptionPolicy.IsPreemptionRequestFor( + getAppAttemptId(1)))); + verify(mDisp, times(0)).handle(argThat( + new TestProportionalCapacityPreemptionPolicy.IsPreemptionRequestFor( + getAppAttemptId(2)))); + } + + @Test + public void testNoIntraQueuePreemptionIfBelowUserLimitAndSamePriorityExtraUsers() + throws IOException { + /** + * Queue structure is: + * + *
+     *       root
+     *        |
+     *        a
+     * 
+ * + * Scenario: + * Preconditions: + * Queue total resources: 100 + * Minimum user limit percent: 50% + * +--------------+----------+------+---------+ + * | APP | USER | PRIORITY | USED | PENDING | + * +--------------+----------+------+---------+ + * | app1 | user1 | 1 | 50 | 0 | + * | app2 | user2 | 1 | 50 | 0 | + * | app3 | user3 | 1 | 0 | 10 | + * +--------------+----------+------+---------+ + * This scenario should never preempt from either user1 or user2 + */ + + // Set max preemption per round to 50% (this is different from minimum user + // limit percent). + conf.setFloat(CapacitySchedulerConfiguration. + INTRAQUEUE_PREEMPTION_MAX_ALLOWABLE_LIMIT, + (float) 0.7); + + String labelsConfig = "=100,true;"; + String nodesConfig = // n1 has no label + "n1= res=100"; + String queuesConfig = + // guaranteed,max,used,pending,reserved + "root(=[100 100 100 10 0]);" + // root + "-a(=[100 100 100 10 0])"; // a + + String appsConfig = + // queueName\t\ + // (priority,resource,host,label,#repeat,reserved,pending,user)\tMULP; + "a\t(1,1,n1,,50,false,0,user1)\t50;" + // app1, user1 + "a\t(1,1,n1,,50,false,0,user2)\t50;" + // app2, user2 + "a\t(1,1,n1,,0,false,10,user3)\t50"; // app3, user3 + + buildEnv(labelsConfig, nodesConfig, queuesConfig, appsConfig); + policy.editSchedule(); + + // app2/app4 needs more resource and its well under its user-limit. Hence + // preempt resources from app1 (compare to app3, app1 has low priority). + verify(mDisp, times(0)).handle(argThat( + new TestProportionalCapacityPreemptionPolicy.IsPreemptionRequestFor( + getAppAttemptId(1)))); + verify(mDisp, times(0)).handle(argThat( + new TestProportionalCapacityPreemptionPolicy.IsPreemptionRequestFor( + getAppAttemptId(2)))); + } + + @Test + public void testNoIntraQueuePreemptionIfBelowUserLimitAndHighPriorityExtraUsers() + throws IOException { + /** + * Queue structure is: + * + *
+     *       root
+     *        |
+     *        a
+     * 
+ * + * Scenario: + * Preconditions: + * Queue total resources: 100 + * Minimum user limit percent: 50% + * +--------------+----------+------+---------+ + * | APP | USER | PRIORITY | USED | PENDING | + * +--------------+----------+------+---------+ + * | app1 | user1 | 1 | 50 | 0 | + * | app2 | user2 | 1 | 50 | 0 | + * | app3 | user3 | 5 | 0 | 10 | + * +--------------+----------+------+---------+ + * This scenario should never preempt from either user1 or user2 + */ + + // Set max preemption per round to 50% (this is different from minimum user + // limit percent). + conf.setFloat(CapacitySchedulerConfiguration. + INTRAQUEUE_PREEMPTION_MAX_ALLOWABLE_LIMIT, + (float) 0.7); + + String labelsConfig = "=100,true;"; + String nodesConfig = // n1 has no label + "n1= res=100"; + String queuesConfig = + // guaranteed,max,used,pending,reserved + "root(=[100 100 100 10 0]);" + // root + "-a(=[100 100 100 10 0])"; // a + + String appsConfig = + // queueName\t\ + // (priority,resource,host,label,#repeat,reserved,pending,user)\tMULP; + "a\t(1,1,n1,,50,false,0,user1)\t50;" + // app1, user1 + "a\t(1,1,n1,,50,false,0,user2)\t50;" + // app2, user2 + "a\t(5,1,n1,,0,false,10,user3)\t50"; // app3, user3 + + buildEnv(labelsConfig, nodesConfig, queuesConfig, appsConfig); + policy.editSchedule(); + + // app2/app4 needs more resource and its well under its user-limit. Hence + // preempt resources from app1 (compare to app3, app1 has low priority). + verify(mDisp, times(0)).handle(argThat( + new TestProportionalCapacityPreemptionPolicy.IsPreemptionRequestFor( + getAppAttemptId(1)))); + verify(mDisp, times(0)).handle(argThat( + new TestProportionalCapacityPreemptionPolicy.IsPreemptionRequestFor( + getAppAttemptId(2)))); + } }