diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/resource/Resources.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/resource/Resources.java index 800fa34..dedc243 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/resource/Resources.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/resource/Resources.java @@ -245,11 +245,16 @@ public static Resource max( Resource lhs, Resource rhs) { return resourceCalculator.compare(clusterResource, lhs, rhs) >= 0 ? lhs : rhs; } - + public static boolean fitsIn(Resource smaller, Resource bigger) { return smaller.getMemory() <= bigger.getMemory() && smaller.getVirtualCores() <= bigger.getVirtualCores(); } + + public static boolean fitsInWithoutEqual(Resource smaller, Resource bigger) { + return smaller.getMemory() < bigger.getMemory() && + smaller.getVirtualCores() < bigger.getVirtualCores(); + } public static Resource componentwiseMin(Resource lhs, Resource rhs) { return createResource(Math.min(lhs.getMemory(), rhs.getMemory()), diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSQueue.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSQueue.java index c265fcf..87f7c13 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSQueue.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSQueue.java @@ -180,7 +180,7 @@ public abstract void collectSchedulerApplications( * @return true if check passes (can assign) or false otherwise */ protected boolean assignContainerPreCheck(FSSchedulerNode node) { - if (!Resources.fitsIn(getResourceUsage(), + if (!Resources.fitsInWithoutEqual(getResourceUsage(), scheduler.getAllocationConfiguration().getMaxResources(getName())) || node.getReservedContainer() != null) { return false; diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFSQueue.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFSQueue.java new file mode 100644 index 0000000..2fe8d04 --- /dev/null +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFSQueue.java @@ -0,0 +1,114 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair; + +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.yarn.api.records.Resource; +import org.apache.hadoop.yarn.conf.YarnConfiguration; +import org.apache.hadoop.yarn.event.AsyncDispatcher; +import org.apache.hadoop.yarn.server.resourcemanager.MockNodes; +import org.apache.hadoop.yarn.server.resourcemanager.ResourceManager; +import org.apache.hadoop.yarn.server.resourcemanager.rmnode.RMNode; +import org.apache.hadoop.yarn.server.resourcemanager.scheduler + .ResourceScheduler; +import org.apache.hadoop.yarn.util.resource.Resources; +import org.junit.Before; +import org.junit.Test; +import org.mockito.Mockito; + +import java.io.IOException; + +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertTrue; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; + +public class TestFSQueue { + private FSLeafQueue leafQueue = null; + private FSParentQueue parentQueue = null; + private Resource maxResource = Resources.createResource(10); + + @Before + public void setup() throws IOException { + FairScheduler scheduler = new FairScheduler(); + Configuration conf = createConfiguration(); + // All tests assume only one assignment per node update + conf.set(FairSchedulerConfiguration.ASSIGN_MULTIPLE, "false"); + ResourceManager resourceManager = new ResourceManager(); + resourceManager.init(conf); + ((AsyncDispatcher)resourceManager.getRMContext().getDispatcher()).start(); + scheduler.reinitialize(conf, resourceManager.getRMContext()); + scheduler.allocConf = mock(AllocationConfiguration.class); + + String leafQueueName = "root.queue1"; + when(scheduler.allocConf.getMaxResources(leafQueueName)).thenReturn(maxResource); + when(scheduler.allocConf.getMinResources(leafQueueName)).thenReturn(Resources.none()); + + String parentQueueName = "root"; + when(scheduler.allocConf.getMaxResources(parentQueueName)).thenReturn(maxResource); + when(scheduler.allocConf.getMinResources(parentQueueName)).thenReturn(Resources.none()); + + parentQueue = new FSParentQueue(parentQueueName, scheduler, null); + leafQueue = new FSLeafQueue(leafQueueName, scheduler, parentQueue); + parentQueue.addChildQueue(leafQueue); + } + + @Test + public void testAssignContainerPreCheck() { + //Prepare + RMNode fakeNode = MockNodes.newNodeInfo(1, maxResource, 1, "127.0.0.1"); + FSSchedulerNode fakeFSNode = new FSSchedulerNode(fakeNode, true); + + AppSchedulable app = mock(AppSchedulable.class); + Mockito.when(app.getResourceUsage()).thenReturn(maxResource); + + //Test the resourceUsage less than maxResource + assertTrue("ResourceUsage is less than max allowed, " + + "should pass assignContainerPreCheck", + parentQueue.assignContainerPreCheck(fakeFSNode)); + assertTrue("ResourceUsage is less than max allowed, " + + "should pass assignContainerPreCheck", + leafQueue.assignContainerPreCheck(fakeFSNode)); + + //Test the resourceUsage equal to maxResource + leafQueue.addAppSchedulable(app); + assertFalse("ResourceUsage is equal to max allowed, " + + "should not pass assignContainerPreCheck", + parentQueue.assignContainerPreCheck(fakeFSNode)); + assertFalse("ResourceUsage is equal to max allowed, " + + "should not pass assignContainerPreCheck", + leafQueue.assignContainerPreCheck(fakeFSNode)); + + //Test the resourceUsage great than maxResource + leafQueue.addAppSchedulable(app); + assertFalse("ResourceUsage is greater than max allowed, " + + "should not pass assignContainerPreCheck", + parentQueue.assignContainerPreCheck(fakeFSNode)); + assertFalse("ResourceUsage is greater than max allowed, " + + "should not pass assignContainerPreCheck", + leafQueue.assignContainerPreCheck(fakeFSNode)); + } + + private Configuration createConfiguration() { + Configuration conf = new YarnConfiguration(); + conf.setClass(YarnConfiguration.RM_SCHEDULER, FairScheduler.class, + ResourceScheduler.class); + return conf; + } +}