diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/resource/DominantResourceCalculator.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/resource/DominantResourceCalculator.java index 6f5b40e..2ee95ce 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/resource/DominantResourceCalculator.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/resource/DominantResourceCalculator.java @@ -53,6 +53,21 @@ public int compare(Resource clusterResource, Resource lhs, Resource rhs) { return 0; } + if (isInvalidDivisor(clusterResource)) { + if ((lhs.getMemory() < rhs.getMemory() && lhs.getVirtualCores() > rhs + .getVirtualCores()) + || (lhs.getMemory() > rhs.getMemory() && lhs.getVirtualCores() < rhs + .getVirtualCores())) { + return 0; + } else if (lhs.getMemory() > rhs.getMemory() + || lhs.getVirtualCores() > rhs.getVirtualCores()) { + return 1; + } else if (lhs.getMemory() < rhs.getMemory() + || lhs.getVirtualCores() < rhs.getVirtualCores()) { + return -1; + } + } + float l = getResourceAsValue(clusterResource, lhs, true); float r = getResourceAsValue(clusterResource, rhs, true); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/util/resource/TestResourceCalculator.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/util/resource/TestResourceCalculator.java new file mode 100644 index 0000000..6a0b62e --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/util/resource/TestResourceCalculator.java @@ -0,0 +1,125 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.yarn.util.resource; + +import java.util.Arrays; +import java.util.Collection; + +import org.apache.hadoop.yarn.api.records.Resource; +import org.junit.Assert; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.junit.runners.Parameterized; + +@RunWith(Parameterized.class) +public class TestResourceCalculator { + private ResourceCalculator resourceCalculator; + + @Parameterized.Parameters + public static Collection getParameters() { + return Arrays.asList(new ResourceCalculator[][] { + { new DefaultResourceCalculator() }, + { new DominantResourceCalculator() } }); + } + + public TestResourceCalculator(ResourceCalculator rs) { + this.resourceCalculator = rs; + } + + @Test(timeout = 10000) + public void testResourceCalculatorCompareMethod() { + Resource clusterResource = Resource.newInstance(0, 0); + + // For lhs == rhs + Resource lhs = Resource.newInstance(0, 0); + Resource rhs = Resource.newInstance(0, 0); + assertResourcesOperations(clusterResource, lhs, rhs, false, true, false, + true, lhs, lhs); + + // lhs > rhs + lhs = Resource.newInstance(1, 1); + rhs = Resource.newInstance(0, 0); + assertResourcesOperations(clusterResource, lhs, rhs, false, false, true, + true, lhs, rhs); + + // For lhs < rhs + lhs = Resource.newInstance(0, 0); + rhs = Resource.newInstance(1, 1); + assertResourcesOperations(clusterResource, lhs, rhs, true, true, false, + false, rhs, lhs); + + if (!(resourceCalculator instanceof DominantResourceCalculator)) { + return; + } + + // verify for 2 dimensional resources i.e memory and cpu + // dominant resource types + lhs = Resource.newInstance(1, 0); + rhs = Resource.newInstance(0, 1); + assertResourcesOperations(clusterResource, lhs, rhs, false, true, false, + true, lhs, lhs); + + lhs = Resource.newInstance(0, 1); + rhs = Resource.newInstance(1, 0); + assertResourcesOperations(clusterResource, lhs, rhs, false, true, false, + true, lhs, lhs); + + lhs = Resource.newInstance(1, 1); + rhs = Resource.newInstance(1, 0); + assertResourcesOperations(clusterResource, lhs, rhs, false, false, true, + true, lhs, rhs); + + lhs = Resource.newInstance(0, 1); + rhs = Resource.newInstance(1, 1); + assertResourcesOperations(clusterResource, lhs, rhs, true, true, false, + false, rhs, lhs); + + } + + + private void assertResourcesOperations(Resource clusterResource, + Resource lhs, Resource rhs, boolean lessThan, boolean lessThanOrEqual, + boolean greaterThan, boolean greaterThanOrEqual, Resource max, + Resource min) { + + Assert.assertEquals("Less Than operation is wrongly calculated.", lessThan, + Resources.lessThan(resourceCalculator, clusterResource, lhs, rhs)); + + Assert.assertEquals( + "Less Than Or Equal To operation is wrongly calculated.", + lessThanOrEqual, Resources.lessThanOrEqual(resourceCalculator, + clusterResource, lhs, rhs)); + + Assert.assertEquals("Greater Than operation is wrongly calculated.", + greaterThan, + Resources.greaterThan(resourceCalculator, clusterResource, lhs, rhs)); + + Assert.assertEquals( + "Greater Than Or Equal To operation is wrongly calculated.", + greaterThanOrEqual, Resources.greaterThanOrEqual(resourceCalculator, + clusterResource, lhs, rhs)); + + Assert.assertEquals("Max(value) Operation wrongly calculated.", max, + Resources.max(resourceCalculator, clusterResource, lhs, rhs)); + + Assert.assertEquals("Min(value) operation is wrongly calculated.", min, + Resources.min(resourceCalculator, clusterResource, lhs, rhs)); + } + +} \ No newline at end of file diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacityScheduler.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacityScheduler.java index 0361424..4d2818c 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacityScheduler.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacityScheduler.java @@ -130,6 +130,7 @@ import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.CapacitySchedulerQueueInfoList; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.policy.FairOrderingPolicy; import org.apache.hadoop.yarn.server.utils.BuilderUtils; +import org.apache.hadoop.yarn.util.resource.DefaultResourceCalculator; import org.apache.hadoop.yarn.util.resource.DominantResourceCalculator; import org.apache.hadoop.yarn.util.resource.Resources; import org.junit.After; @@ -1281,9 +1282,15 @@ public void testRecoverRequestAfterPreemption() throws Exception { private MockRM setUpMove() { CapacitySchedulerConfiguration conf = new CapacitySchedulerConfiguration(); + return setUpMove(conf); + } + + private MockRM setUpMove(Configuration config) { + CapacitySchedulerConfiguration conf = + new CapacitySchedulerConfiguration(config); setupQueueConfiguration(conf); conf.setClass(YarnConfiguration.RM_SCHEDULER, CapacityScheduler.class, - ResourceScheduler.class); + ResourceScheduler.class); MockRM rm = new MockRM(conf); rm.start(); return rm; @@ -2952,6 +2959,55 @@ public void testDefaultNodeLabelExpressionQueueConfig() throws Exception { Assert.assertEquals(queueInfoB.getDefaultNodeLabelExpression(), "y"); } + @Test(timeout = 30000) + public void testAMLimitUsage() throws Exception { + + CapacitySchedulerConfiguration config = + new CapacitySchedulerConfiguration(); + + config.set(CapacitySchedulerConfiguration.RESOURCE_CALCULATOR_CLASS, + DefaultResourceCalculator.class.getName()); + verifyAMLimitForLeafQueue(config); + + config.set(CapacitySchedulerConfiguration.RESOURCE_CALCULATOR_CLASS, + DominantResourceCalculator.class.getName()); + verifyAMLimitForLeafQueue(config); + + } + + private void verifyAMLimitForLeafQueue(CapacitySchedulerConfiguration config) + throws Exception { + MockRM rm = setUpMove(config); + + String queueName = "a1"; + String userName = "user_0"; + ResourceScheduler scheduler = rm.getRMContext().getScheduler(); + LeafQueue queueA = + (LeafQueue) ((CapacityScheduler) scheduler).getQueue(queueName); + Resource amResourceLimit = queueA.getAMResourceLimit(); + + Resource amResource = + Resource.newInstance(amResourceLimit.getMemory() + 1, + amResourceLimit.getVirtualCores() + 1); + + rm.submitApp(amResource.getMemory(), "app-1", userName, null, queueName); + + rm.submitApp(amResource.getMemory(), "app-1", userName, null, queueName); + + // When AM limit is exceeded, 1 applications will be activated.Rest all + // applications will be in pending + Assert.assertEquals("PendingApplications should be 1", 1, + queueA.getNumPendingApplications()); + Assert.assertEquals("Active applications should be 1", 1, + queueA.getNumActiveApplications()); + + Assert.assertEquals("User PendingApplications should be 1", 1, queueA + .getUser(userName).getPendingApplications()); + Assert.assertEquals("User Active applications should be 1", 1, queueA + .getUser(userName).getActiveApplications()); + rm.stop(); + } + private void setMaxAllocMb(Configuration conf, int maxAllocMb) { conf.setInt(YarnConfiguration.RM_SCHEDULER_MAXIMUM_ALLOCATION_MB, maxAllocMb); -- 1.9.2.msysgit.0