diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestRM.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestRM.java index cd67ebc..c161239 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestRM.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestRM.java @@ -22,7 +22,13 @@ import static org.mockito.Mockito.doNothing; import static org.mockito.Mockito.spy; +import java.io.File; +import java.io.FileWriter; +import java.io.IOException; +import java.io.PrintWriter; import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collection; import java.util.EnumSet; import java.util.HashMap; import java.util.List; @@ -30,6 +36,9 @@ import org.apache.hadoop.metrics2.lib.DefaultMetricsSystem; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.QueueMetrics; +import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.FairScheduler; +import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.FairSchedulerConfiguration; +import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fifo.FifoScheduler; import org.junit.After; import org.junit.Assert; @@ -65,23 +74,82 @@ import org.apache.hadoop.yarn.server.resourcemanager.rmapp.attempt.RMAppAttemptEventType; import org.apache.hadoop.yarn.server.resourcemanager.rmapp.attempt.RMAppAttemptState; import org.apache.hadoop.yarn.server.resourcemanager.rmnode.RMNode; -import org.apache.hadoop.yarn.server.resourcemanager.scheduler.QueueMetrics; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacityScheduler; import org.apache.hadoop.yarn.server.resourcemanager.security.NMTokenSecretManagerInRM; import org.apache.log4j.Level; import org.apache.log4j.LogManager; import org.apache.log4j.Logger; +import org.junit.Before; import org.junit.Test; +import org.junit.runner.RunWith; +import org.junit.runners.Parameterized; +import org.junit.runners.Parameterized.Parameters; import org.mockito.ArgumentMatcher; @SuppressWarnings({"unchecked", "rawtypes"}) +@RunWith(Parameterized.class) public class TestRM { + private final static String TEST_DIR = + new File(System.getProperty("test.build.data", "/tmp")).getAbsolutePath(); + private final static String FS_ALLOC_FILE = + new File(TEST_DIR, "test-queues").getAbsolutePath(); private static final Log LOG = LogFactory.getLog(TestRM.class); // Milliseconds to sleep for when waiting for something to happen private final static int WAIT_SLEEP_MS = 100; + private int schedulerType = 0; + private YarnConfiguration conf = null; + + @Parameters + public static Collection configRunnings() { + return Arrays.asList(new Object[][]{{0}, {1}, {2}}); + } + + public TestRM(int para) { + this.schedulerType = para; + } + + @Before + public void setup() throws IOException { + conf = new YarnConfiguration(); + + // Configure scheduler + switch (schedulerType) { + case 0: + configFifoScheduler(); + break; + case 1: + configCapacityScheduler(); + break; + case 2: + configFairScheduler(); + break; + } + } + + private void configFifoScheduler() { + conf.set(YarnConfiguration.RM_SCHEDULER, FifoScheduler.class.getName()); + } + + private void configCapacityScheduler() { + conf.set(YarnConfiguration.RM_SCHEDULER, CapacityScheduler.class.getName()); + } + + private void configFairScheduler() throws IOException { + // Disable queueMaxAMShare limitation for fair scheduler + PrintWriter out = new PrintWriter(new FileWriter(FS_ALLOC_FILE)); + out.println(""); + out.println(""); + out.println("-1.0"); + out.println(""); + out.close(); + + conf.set(YarnConfiguration.RM_SCHEDULER, FairScheduler.class.getName()); + conf.set(FairSchedulerConfiguration.ALLOCATION_FILE, FS_ALLOC_FILE); + } + @After public void tearDown() { ClusterMetrics.destroy(); @@ -93,7 +161,7 @@ public void tearDown() { public void testGetNewAppId() throws Exception { Logger rootLogger = LogManager.getRootLogger(); rootLogger.setLevel(Level.DEBUG); - MockRM rm = new MockRM(); + MockRM rm = new MockRM(conf); rm.start(); GetNewApplicationResponse resp = rm.getNewAppId(); @@ -106,7 +174,7 @@ public void testGetNewAppId() throws Exception { public void testAppWithNoContainers() throws Exception { Logger rootLogger = LogManager.getRootLogger(); rootLogger.setLevel(Level.DEBUG); - MockRM rm = new MockRM(); + MockRM rm = new MockRM(conf); rm.start(); MockNM nm1 = rm.registerNode("h1:1234", 5120); @@ -128,7 +196,6 @@ public void testAppWithNoContainers() throws Exception { public void testAppOnMultiNode() throws Exception { Logger rootLogger = LogManager.getRootLogger(); rootLogger.setLevel(Level.DEBUG); - YarnConfiguration conf = new YarnConfiguration(); conf.set("yarn.scheduler.capacity.node-locality-delay", "-1"); MockRM rm = new MockRM(conf); rm.start(); @@ -188,7 +255,6 @@ public void testAppOnMultiNode() throws Exception { // corresponding NM Token. @Test (timeout = 20000) public void testNMTokenSentForNormalContainer() throws Exception { - YarnConfiguration conf = new YarnConfiguration(); conf.set(YarnConfiguration.RM_SCHEDULER, CapacityScheduler.class.getCanonicalName()); MockRM rm = new MockRM(conf); @@ -240,7 +306,7 @@ public void testNMTokenSentForNormalContainer() throws Exception { @Test (timeout = 40000) public void testNMToken() throws Exception { - MockRM rm = new MockRM(); + MockRM rm = new MockRM(conf); try { rm.start(); MockNM nm1 = rm.registerNode("h1:1234", 10000); @@ -422,7 +488,12 @@ protected void allocateContainersAndValidateNMTokens(MockAM am, @Test (timeout = 300000) public void testActivatingApplicationAfterAddingNM() throws Exception { - YarnConfiguration conf = new YarnConfiguration(); + // This test doesn't work for FifoScheduler, as FifoScheduler doesn't + // support ASSIGN_MULTIPLE setting; that is, FifoScheduler tries to assign + // as many as possible containers per node heartbeat. + if (schedulerType == 0) { + return; + } MockRM rm1 = new MockRM(conf); @@ -469,7 +540,6 @@ public void testActivatingApplicationAfterAddingNM() throws Exception { // is killed or failed, so that client doesn't get the wrong information. @Test (timeout = 80000) public void testInvalidateAMHostPortWhenAMFailedOrKilled() throws Exception { - YarnConfiguration conf = new YarnConfiguration(); conf.setInt(YarnConfiguration.RM_AM_MAX_ATTEMPTS, 1); MockRM rm1 = new MockRM(conf); rm1.start(); @@ -522,7 +592,6 @@ public void testInvalidateAMHostPortWhenAMFailedOrKilled() throws Exception { @Test (timeout = 60000) public void testInvalidatedAMHostPortOnAMRestart() throws Exception { - YarnConfiguration conf = new YarnConfiguration(); MockRM rm1 = new MockRM(conf); rm1.start(); MockNM nm1 = @@ -555,7 +624,6 @@ public void testInvalidatedAMHostPortOnAMRestart() throws Exception { @Test (timeout = 60000) public void testApplicationKillAtAcceptedState() throws Exception { - YarnConfiguration conf = new YarnConfiguration(); final Dispatcher dispatcher = new AsyncDispatcher() { @Override public EventHandler getEventHandler() { @@ -632,15 +700,4 @@ protected Dispatcher createDispatcher() { Assert.assertEquals(appsSubmitted + 1, metrics.getAppsSubmitted()); } - public static void main(String[] args) throws Exception { - TestRM t = new TestRM(); - t.testGetNewAppId(); - t.testAppWithNoContainers(); - t.testAppOnMultiNode(); - t.testNMToken(); - t.testActivatingApplicationAfterAddingNM(); - t.testInvalidateAMHostPortWhenAMFailedOrKilled(); - t.testInvalidatedAMHostPortOnAMRestart(); - t.testApplicationKillAtAcceptedState(); - } }