diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/reservation/ReservationSystemTestUtil.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/reservation/ReservationSystemTestUtil.java index be1d69a..36a2a56 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/reservation/ReservationSystemTestUtil.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/reservation/ReservationSystemTestUtil.java @@ -72,11 +72,11 @@ public static ReservationId getNewReservationId() { public static ReservationSchedulerConfiguration createConf( String reservationQ, long timeWindow, float instConstraint, float avgConstraint) { - ReservationSchedulerConfiguration conf = mock - (ReservationSchedulerConfiguration.class); + ReservationSchedulerConfiguration conf = + mock(ReservationSchedulerConfiguration.class); when(conf.getReservationWindow(reservationQ)).thenReturn(timeWindow); - when(conf.getInstantaneousMaxCapacity(reservationQ)).thenReturn - (instConstraint); + when(conf.getInstantaneousMaxCapacity(reservationQ)).thenReturn( + instConstraint); when(conf.getAverageCapacity(reservationQ)).thenReturn(avgConstraint); return conf; } @@ -88,10 +88,10 @@ public static void validateReservationQueue( Assert.assertTrue(plan instanceof InMemoryPlan); Assert.assertEquals(planQName, plan.getQueueName()); Assert.assertEquals(8192, plan.getTotalCapacity().getMemory()); - Assert.assertTrue( - plan.getReservationAgent() instanceof GreedyReservationAgent); - Assert.assertTrue( - plan.getSharingPolicy() instanceof CapacityOverTimePolicy); + Assert + .assertTrue(plan.getReservationAgent() instanceof GreedyReservationAgent); + Assert + .assertTrue(plan.getSharingPolicy() instanceof CapacityOverTimePolicy); } public static void validateNewReservationQueue( @@ -152,21 +152,19 @@ public static void updateFSAllocationFile(String allocationFile) out.println(""); out.println(""); out.println(""); - out.println("80"); + out.println("10"); out.println(""); out.println(""); out.println(""); - out.println("10"); + out.println("80"); out.println(""); out.println("drf"); out.println(""); out.close(); } - public static FairScheduler setupFairScheduler( - ReservationSystemTestUtil testUtil, - RMContext rmContext, Configuration conf, int numContainers) throws - IOException { + public static FairScheduler setupFairScheduler(RMContext rmContext, + Configuration conf, int numContainers) throws IOException { FairScheduler scheduler = new FairScheduler(); scheduler.setRMContext(rmContext); @@ -176,15 +174,14 @@ public static FairScheduler setupFairScheduler( scheduler.start(); scheduler.reinitialize(conf, rmContext); - - Resource resource = testUtil.calculateClusterResource(numContainers); + Resource resource = + ReservationSystemTestUtil.calculateClusterResource(numContainers); RMNode node1 = MockNodes.newNodeInfo(1, resource, 1, "127.0.0.1"); NodeAddedSchedulerEvent nodeEvent1 = new NodeAddedSchedulerEvent(node1); scheduler.handle(nodeEvent1); return scheduler; } - @SuppressWarnings("unchecked") public CapacityScheduler mockCapacityScheduler(int numContainers) throws IOException { // stolen from TestCapacityScheduler @@ -207,6 +204,7 @@ public CapacityScheduler mockCapacityScheduler(int numContainers) return cs; } + @SuppressWarnings("rawtypes") public static void initializeRMContext(int numContainers, AbstractYarnScheduler scheduler, RMContext mockRMContext) { @@ -273,16 +271,16 @@ public static void setupQueueConfiguration(CapacitySchedulerConfiguration conf) conf.setCapacity(A2, 70); } - public String getFullReservationQueueName() { + public static String getFullReservationQueueName() { return CapacitySchedulerConfiguration.ROOT + CapacitySchedulerConfiguration.DOT + reservationQ; } - public String getreservationQueueName() { + public static String getreservationQueueName() { return reservationQ; } - public void updateQueueConfiguration(CapacitySchedulerConfiguration conf, + public static void updateQueueConfiguration(CapacitySchedulerConfiguration conf, String newQ) { // Define default queue final String prefix = @@ -299,18 +297,18 @@ public void updateQueueConfiguration(CapacitySchedulerConfiguration conf, conf.setCapacity(A, 5); final String dedicated = prefix + reservationQ; - conf.setCapacity(dedicated, 80); + conf.setCapacity(dedicated, 10); // Set as reservation queue conf.setReservable(dedicated, true); - conf.setCapacity(prefix + newQ, 10); + conf.setCapacity(prefix + newQ, 80); // Set as reservation queue conf.setReservable(prefix + newQ, true); // Define 2nd-level queues final String A1 = A + ".a1"; final String A2 = A + ".a2"; - conf.setQueues(A, new String[]{"a1", "a2"}); + conf.setQueues(A, new String[] { "a1", "a2" }); conf.setCapacity(A1, 30); conf.setCapacity(A2, 70); } @@ -385,15 +383,14 @@ public static ReservationDefinition generateBigRR(Random rand, long i) { for (int i = 0; i < alloc.length; i++) { req.put(new ReservationInterval(startTime + i * step, startTime + (i + 1) * step), ReservationSystemUtil.toResource(ReservationRequest - .newInstance( - Resource.newInstance(1024, 1), alloc[i]))); + .newInstance(Resource.newInstance(1024, 1), alloc[i]))); } return req; } public static Resource calculateClusterResource(int numContainers) { - Resource clusterResource = Resource.newInstance(numContainers * 1024, - numContainers); + Resource clusterResource = + Resource.newInstance(numContainers * 1024, numContainers); return clusterResource; } } diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/reservation/TestCapacityReservationSystem.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/reservation/TestCapacityReservationSystem.java deleted file mode 100644 index 11e52c4..0000000 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/reservation/TestCapacityReservationSystem.java +++ /dev/null @@ -1,94 +0,0 @@ -/******************************************************************************* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - *******************************************************************************/ -package org.apache.hadoop.yarn.server.resourcemanager.reservation; - -import java.io.IOException; - -import org.apache.hadoop.yarn.exceptions.YarnException; -import org.apache.hadoop.yarn.server.resourcemanager.RMContext; -import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacityScheduler; -import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacitySchedulerConfiguration; -import org.junit.Assert; -import org.junit.Test; - -public class TestCapacityReservationSystem { - - @Test - public void testInitialize() { - ReservationSystemTestUtil testUtil = new ReservationSystemTestUtil(); - CapacityScheduler capScheduler = null; - try { - capScheduler = testUtil.mockCapacityScheduler(10); - } catch (IOException e) { - Assert.fail(e.getMessage()); - } - CapacityReservationSystem reservationSystem = - new CapacityReservationSystem(); - reservationSystem.setRMContext(capScheduler.getRMContext()); - try { - reservationSystem.reinitialize(capScheduler.getConf(), - capScheduler.getRMContext()); - } catch (YarnException e) { - Assert.fail(e.getMessage()); - } - String planQName = testUtil.getreservationQueueName(); - ReservationSystemTestUtil.validateReservationQueue(reservationSystem, - planQName); - } - - @Test - public void testReinitialize() { - ReservationSystemTestUtil testUtil = new ReservationSystemTestUtil(); - CapacityScheduler capScheduler = null; - try { - capScheduler = testUtil.mockCapacityScheduler(10); - } catch (IOException e) { - Assert.fail(e.getMessage()); - } - CapacityReservationSystem reservationSystem = - new CapacityReservationSystem(); - CapacitySchedulerConfiguration conf = capScheduler.getConfiguration(); - RMContext mockContext = capScheduler.getRMContext(); - reservationSystem.setRMContext(mockContext); - try { - reservationSystem.reinitialize(capScheduler.getConfiguration(), - mockContext); - } catch (YarnException e) { - Assert.fail(e.getMessage()); - } - // Assert queue in original config - String planQName = testUtil.getreservationQueueName(); - ReservationSystemTestUtil.validateReservationQueue(reservationSystem, planQName); - - // Dynamically add a plan - String newQ = "reservation"; - Assert.assertNull(reservationSystem.getPlan(newQ)); - testUtil.updateQueueConfiguration(conf, newQ); - try { - capScheduler.reinitialize(conf, mockContext); - } catch (IOException e) { - Assert.fail(e.getMessage()); - } - try { - reservationSystem.reinitialize(conf, mockContext); - } catch (YarnException e) { - Assert.fail(e.getMessage()); - } - ReservationSystemTestUtil.validateNewReservationQueue(reservationSystem, newQ); - } -} diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/reservation/TestCapacitySchedulerPlanFollower.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/reservation/TestCapacitySchedulerPlanFollower.java index b8663f6..88f351e 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/reservation/TestCapacitySchedulerPlanFollower.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/reservation/TestCapacitySchedulerPlanFollower.java @@ -56,7 +56,8 @@ import org.mockito.Matchers; import org.mockito.Mockito; -public class TestCapacitySchedulerPlanFollower extends TestSchedulerPlanFollowerBase { +public class TestCapacitySchedulerPlanFollower extends + TestSchedulerPlanFollowerBase { private RMContext rmContext; private RMContext spyRMContext; @@ -115,11 +116,11 @@ public void setUp() throws Exception { } private void setupPlanFollower() throws Exception { - ReservationSystemTestUtil testUtil = new ReservationSystemTestUtil(); mClock = mock(Clock.class); mAgent = mock(ReservationAgent.class); - String reservationQ = testUtil.getFullReservationQueueName(); + String reservationQ = + ReservationSystemTestUtil.getFullReservationQueueName(); CapacitySchedulerConfiguration csConf = cs.getConfiguration(); csConf.setReservationWindow(reservationQ, 20L); csConf.setMaximumCapacity(reservationQ, 40); @@ -143,7 +144,7 @@ public void testWithKillOnExpiry() throws PlanningException, @Override protected void verifyCapacity(Queue defQ) { - CSQueue csQueue = (CSQueue)defQ; + CSQueue csQueue = (CSQueue) defQ; assertTrue(csQueue.getCapacity() > 0.9); } @@ -154,7 +155,7 @@ protected Queue getDefaultQueue() { @Override protected int getNumberOfApplications(Queue queue) { - CSQueue csQueue = (CSQueue)queue; + CSQueue csQueue = (CSQueue) queue; int numberOfApplications = csQueue.getNumApplications(); return numberOfApplications; } diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/reservation/TestFairReservationSystem.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/reservation/TestFairReservationSystem.java deleted file mode 100644 index f294eaf..0000000 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/reservation/TestFairReservationSystem.java +++ /dev/null @@ -1,128 +0,0 @@ -/******************************************************************************* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - *******************************************************************************/ -package org.apache.hadoop.yarn.server.resourcemanager.reservation; - -import org.apache.hadoop.conf.Configuration; -import org.apache.hadoop.yarn.conf.YarnConfiguration; -import org.apache.hadoop.yarn.exceptions.YarnException; -import org.apache.hadoop.yarn.server.resourcemanager.MockNodes; -import org.apache.hadoop.yarn.server.resourcemanager.RMContext; -import org.apache.hadoop.yarn.server.resourcemanager.scheduler.ResourceScheduler; -import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.FairScheduler; -import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.FairSchedulerConfiguration; -import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.FairSchedulerTestBase; - -import org.junit.After; -import org.junit.Assert; -import org.junit.Before; -import org.junit.Test; - -import java.io.File; -import java.io.IOException; - -public class TestFairReservationSystem { - private final static String ALLOC_FILE = new File(FairSchedulerTestBase. - TEST_DIR, - TestFairReservationSystem.class.getName() + ".xml").getAbsolutePath(); - private Configuration conf; - private FairScheduler scheduler; - private FairSchedulerTestBase testHelper = new FairSchedulerTestBase(); - - protected Configuration createConfiguration() { - Configuration conf = testHelper.createConfiguration(); - conf.setClass(YarnConfiguration.RM_SCHEDULER, FairScheduler.class, - ResourceScheduler.class); - conf.set(FairSchedulerConfiguration.ALLOCATION_FILE, ALLOC_FILE); - return conf; - } - - @Before - public void setup() throws IOException { - conf = createConfiguration(); - } - - @After - public void teardown() { - conf = null; - } - - @Test - public void testFairReservationSystemInitialize() throws IOException { - ReservationSystemTestUtil.setupFSAllocationFile(ALLOC_FILE); - - ReservationSystemTestUtil testUtil = new ReservationSystemTestUtil(); - - // Setup - RMContext mockRMContext = testUtil.createRMContext(conf); - scheduler = ReservationSystemTestUtil.setupFairScheduler(testUtil, - mockRMContext, conf, 10); - - FairReservationSystem reservationSystem = new FairReservationSystem(); - reservationSystem.setRMContext(mockRMContext); - - try { - reservationSystem.reinitialize(scheduler.getConf(), mockRMContext); - } catch (YarnException e) { - Assert.fail(e.getMessage()); - } - - ReservationSystemTestUtil.validateReservationQueue(reservationSystem, - testUtil.getFullReservationQueueName()); - } - - @Test - public void testFairReservationSystemReinitialize() throws IOException { - ReservationSystemTestUtil.setupFSAllocationFile(ALLOC_FILE); - - ReservationSystemTestUtil testUtil = new ReservationSystemTestUtil(); - - // Setup - RMContext mockRMContext = testUtil.createRMContext(conf); - scheduler = ReservationSystemTestUtil.setupFairScheduler(testUtil, - mockRMContext, conf, 10); - - FairReservationSystem reservationSystem = new FairReservationSystem(); - reservationSystem.setRMContext(mockRMContext); - - try { - reservationSystem.reinitialize(scheduler.getConf(), mockRMContext); - } catch (YarnException e) { - Assert.fail(e.getMessage()); - } - - // Assert queue in original config - final String planQNam = testUtil.getFullReservationQueueName(); - ReservationSystemTestUtil.validateReservationQueue(reservationSystem, - planQNam); - - // Dynamically add a plan - ReservationSystemTestUtil.updateFSAllocationFile(ALLOC_FILE); - scheduler.reinitialize(conf, mockRMContext); - - try { - reservationSystem.reinitialize(conf, mockRMContext); - } catch (YarnException e) { - Assert.fail(e.getMessage()); - } - - String newQueue = "root.reservation"; - ReservationSystemTestUtil.validateNewReservationQueue - (reservationSystem, newQueue); - } - -} diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/reservation/TestFairSchedulerPlanFollower.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/reservation/TestFairSchedulerPlanFollower.java index e9a4f50..0694ff1 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/reservation/TestFairSchedulerPlanFollower.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/reservation/TestFairSchedulerPlanFollower.java @@ -1,20 +1,20 @@ /** -* Licensed to the Apache Software Foundation (ASF) under one -* or more contributor license agreements. See the NOTICE file -* distributed with this work for additional information -* regarding copyright ownership. The ASF licenses this file -* to you under the Apache License, Version 2.0 (the -* "License"); you may not use this file except in compliance -* with the License. You may obtain a copy of the License at -* -* http://www.apache.org/licenses/LICENSE-2.0 -* -* Unless required by applicable law or agreed to in writing, software -* distributed under the License is distributed on an "AS IS" BASIS, -* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -* See the License for the specific language governing permissions and -* limitations under the License. -*/ + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ package org.apache.hadoop.yarn.server.resourcemanager.reservation; import static org.junit.Assert.assertNotNull; @@ -61,9 +61,9 @@ public class TestFairSchedulerPlanFollower extends TestSchedulerPlanFollowerBase { - private final static String ALLOC_FILE = new File(FairSchedulerTestBase. - TEST_DIR, - TestFairReservationSystem.class.getName() + ".xml").getAbsolutePath(); + private final static String ALLOC_FILE = new File( + FairSchedulerTestBase.TEST_DIR, + TestSchedulerPlanFollowerBase.class.getName() + ".xml").getAbsolutePath(); private RMContext rmContext; private RMContext spyRMContext; private FairScheduler fs; @@ -85,13 +85,11 @@ protected Configuration createConfiguration() { public void setUp() throws Exception { conf = createConfiguration(); ReservationSystemTestUtil.setupFSAllocationFile(ALLOC_FILE); - ReservationSystemTestUtil testUtil = new ReservationSystemTestUtil(); // Setup rmContext = TestUtils.getMockRMContext(); spyRMContext = spy(rmContext); - fs = ReservationSystemTestUtil.setupFairScheduler(testUtil, - spyRMContext, conf, 125); + fs = ReservationSystemTestUtil.setupFairScheduler(spyRMContext, conf, 125); scheduler = fs; ConcurrentMap spyApps = @@ -107,11 +105,11 @@ public void setUp() throws Exception { } private void setupPlanFollower() throws Exception { - ReservationSystemTestUtil testUtil = new ReservationSystemTestUtil(); mClock = mock(Clock.class); mAgent = mock(ReservationAgent.class); - String reservationQ = testUtil.getFullReservationQueueName(); + String reservationQ = + ReservationSystemTestUtil.getFullReservationQueueName(); AllocationConfiguration allocConf = fs.getAllocationConfiguration(); allocConf.setReservationWindow(20L); allocConf.setAverageCapacity(20); @@ -134,14 +132,13 @@ public void testWithKillOnExpiry() throws PlanningException, @Override protected void verifyCapacity(Queue defQ) { - assertTrue(((FSQueue) defQ).getWeights().getWeight(ResourceType.MEMORY) > - 0.9); + assertTrue(((FSQueue) defQ).getWeights().getWeight(ResourceType.MEMORY) > 0.9); } @Override protected Queue getDefaultQueue() { - return getReservationQueue("dedicated" + - ReservationConstants.DEFAULT_QUEUE_SUFFIX); + return getReservationQueue("dedicated" + + ReservationConstants.DEFAULT_QUEUE_SUFFIX); } @Override @@ -152,8 +149,7 @@ protected int getNumberOfApplications(Queue queue) { @Override protected AbstractSchedulerPlanFollower createPlanFollower() { - FairSchedulerPlanFollower planFollower = - new FairSchedulerPlanFollower(); + FairSchedulerPlanFollower planFollower = new FairSchedulerPlanFollower(); planFollower.init(mClock, scheduler, Collections.singletonList(plan)); return planFollower; } @@ -167,13 +163,13 @@ protected void assertReservationQueueExists(ReservationId r) { @Override protected void assertReservationQueueExists(ReservationId r, double expectedCapacity, double expectedMaxCapacity) { - FSLeafQueue q = fs.getQueueManager().getLeafQueue(plan.getQueueName() + "" + - "." + - r, false); + FSLeafQueue q = + fs.getQueueManager().getLeafQueue(plan.getQueueName() + "" + "." + r, + false); assertNotNull(q); // For now we are setting both to same weight - Assert.assertEquals(expectedCapacity, q.getWeights().getWeight - (ResourceType.MEMORY), 0.01); + Assert.assertEquals(expectedCapacity, + q.getWeights().getWeight(ResourceType.MEMORY), 0.01); } @Override @@ -184,9 +180,8 @@ protected void assertReservationQueueDoesNotExist(ReservationId r) { @Override protected Queue getReservationQueue(String r) { - return fs.getQueueManager().getLeafQueue(plan.getQueueName() + "" + - "." + - r, false); + return fs.getQueueManager().getLeafQueue( + plan.getQueueName() + "" + "." + r, false); } public static ApplicationACLsManager mockAppACLsManager() { diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/reservation/TestReservationSystem.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/reservation/TestReservationSystem.java new file mode 100644 index 0000000..c625d7f --- /dev/null +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/reservation/TestReservationSystem.java @@ -0,0 +1,228 @@ +/******************************************************************************* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + *******************************************************************************/ +package org.apache.hadoop.yarn.server.resourcemanager.reservation; + +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.metrics2.lib.DefaultMetricsSystem; +import org.apache.hadoop.yarn.conf.YarnConfiguration; +import org.apache.hadoop.yarn.exceptions.YarnException; +import org.apache.hadoop.yarn.server.resourcemanager.ParameterizedSchedulerTestBase; +import org.apache.hadoop.yarn.server.resourcemanager.RMContext; +import org.apache.hadoop.yarn.server.resourcemanager.scheduler.AbstractYarnScheduler; +import org.apache.hadoop.yarn.server.resourcemanager.scheduler.QueueMetrics; +import org.apache.hadoop.yarn.server.resourcemanager.scheduler.ResourceScheduler; +import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacityScheduler; +import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacitySchedulerConfiguration; +import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.FairScheduler; +import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.FairSchedulerConfiguration; +import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.FairSchedulerTestBase; +import org.junit.After; +import org.junit.Assert; +import org.junit.Before; +import org.junit.Test; +import org.mockito.Mockito; + +import java.io.File; +import java.io.IOException; + +@SuppressWarnings({ "rawtypes" }) +public class TestReservationSystem extends + ParameterizedSchedulerTestBase { + + private final static String ALLOC_FILE = new File( + FairSchedulerTestBase.TEST_DIR, TestReservationSystem.class.getName() + + ".xml").getAbsolutePath(); + private ReservationSystemType reservationSystemType; + private AbstractYarnScheduler scheduler; + private AbstractReservationSystem reservationSystem; + private RMContext rmContext; + private Configuration conf; + private RMContext mockRMContext; + + public TestReservationSystem(SchedulerType type) { + super(type); + if (type == SchedulerType.CAPACITY) { + reservationSystemType = TestReservationSystem.ReservationSystemType.CAPACITY; + } else { + reservationSystemType = TestReservationSystem.ReservationSystemType.FAIR; + } + } + + @Before + public void setUp() throws IOException { + scheduler = initializeScheduler(); + rmContext = getRMContext(); + reservationSystem = configureReservationSystem(); + reservationSystem.setRMContext(rmContext); + DefaultMetricsSystem.setMiniClusterMode(true); + } + + @After + public void tearDown() { + conf = null; + reservationSystem = null; + rmContext = null; + scheduler = null; + clearRMContext(); + QueueMetrics.clearQueueMetrics(); + } + + @Test + public void testInitialize() throws IOException { + try { + reservationSystem.reinitialize(scheduler.getConfig(), rmContext); + } catch (YarnException e) { + Assert.fail(e.getMessage()); + } + if (getReservationSystemType().equals(ReservationSystemType.CAPACITY)) { + ReservationSystemTestUtil.validateReservationQueue(reservationSystem, + ReservationSystemTestUtil.getreservationQueueName()); + } else { + ReservationSystemTestUtil.validateReservationQueue(reservationSystem, + ReservationSystemTestUtil.getFullReservationQueueName()); + } + + } + + @Test + public void testReinitialize() throws IOException { + conf = scheduler.getConfig(); + try { + reservationSystem.reinitialize(conf, rmContext); + } catch (YarnException e) { + Assert.fail(e.getMessage()); + } + if (getReservationSystemType().equals(ReservationSystemType.CAPACITY)) { + ReservationSystemTestUtil.validateReservationQueue(reservationSystem, + ReservationSystemTestUtil.getreservationQueueName()); + } else { + ReservationSystemTestUtil.validateReservationQueue(reservationSystem, + ReservationSystemTestUtil.getFullReservationQueueName()); + } + + // Dynamically add a plan + String newQ = "reservation"; + Assert.assertNull(reservationSystem.getPlan(newQ)); + updateSchedulerConf(conf, newQ); + try { + scheduler.reinitialize(conf, rmContext); + } catch (IOException e) { + Assert.fail(e.getMessage()); + } + try { + reservationSystem.reinitialize(conf, rmContext); + } catch (YarnException e) { + Assert.fail(e.getMessage()); + } + if (getReservationSystemType().equals(ReservationSystemType.CAPACITY)) { + ReservationSystemTestUtil.validateReservationQueue(reservationSystem, + newQ); + } else { + ReservationSystemTestUtil.validateReservationQueue(reservationSystem, + "root." + newQ); + } + } + + @SuppressWarnings("rawtypes") + public AbstractYarnScheduler initializeScheduler() throws IOException { + switch (reservationSystemType) { + case CAPACITY: + return initializeCapacityScheduler(); + case FAIR: + return initializeFairScheduler(); + } + return null; + } + + public AbstractReservationSystem configureReservationSystem() { + switch (reservationSystemType) { + case CAPACITY: + return new CapacityReservationSystem(); + case FAIR: + return new FairReservationSystem(); + } + return null; + } + + public void updateSchedulerConf(Configuration conf, String newQ) + throws IOException { + switch (reservationSystemType) { + case CAPACITY: + ReservationSystemTestUtil.updateQueueConfiguration( + (CapacitySchedulerConfiguration) conf, newQ); + case FAIR: + ReservationSystemTestUtil.updateFSAllocationFile(ALLOC_FILE); + } + } + + public ReservationSystemType getReservationSystemType() { + return reservationSystemType; + } + + public RMContext getRMContext() { + return mockRMContext; + } + + public void clearRMContext() { + mockRMContext = null; + } + + private CapacityScheduler initializeCapacityScheduler() { + // stolen from TestCapacityScheduler + CapacitySchedulerConfiguration conf = new CapacitySchedulerConfiguration(); + ReservationSystemTestUtil.setupQueueConfiguration(conf); + + CapacityScheduler cs = Mockito.spy(new CapacityScheduler()); + cs.setConf(conf); + + mockRMContext = ReservationSystemTestUtil.createRMContext(conf); + + cs.setRMContext(mockRMContext); + try { + cs.serviceInit(conf); + } catch (Exception e) { + Assert.fail(e.getMessage()); + } + ReservationSystemTestUtil.initializeRMContext(10, cs, mockRMContext); + return cs; + } + + private Configuration createFSConfiguration() { + FairSchedulerTestBase testHelper = new FairSchedulerTestBase(); + Configuration conf = testHelper.createConfiguration(); + conf.setClass(YarnConfiguration.RM_SCHEDULER, FairScheduler.class, + ResourceScheduler.class); + conf.set(FairSchedulerConfiguration.ALLOCATION_FILE, ALLOC_FILE); + return conf; + } + + private FairScheduler initializeFairScheduler() throws IOException { + Configuration conf = createFSConfiguration(); + ReservationSystemTestUtil.setupFSAllocationFile(ALLOC_FILE); + + // Setup + mockRMContext = ReservationSystemTestUtil.createRMContext(conf); + FairScheduler fairScheduler = + ReservationSystemTestUtil.setupFairScheduler(mockRMContext, conf, 10); + return fairScheduler; + } + + public enum ReservationSystemType { + CAPACITY, FAIR + } +}