Index: hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/applicationsmanager/TestAMRestart.java IDEA additional info: Subsystem: com.intellij.openapi.diff.impl.patch.CharsetEP <+>UTF-8 =================================================================== --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/applicationsmanager/TestAMRestart.java (date 1511893623000) +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/applicationsmanager/TestAMRestart.java (date 1511915893000) @@ -18,7 +18,9 @@ package org.apache.hadoop.yarn.server.resourcemanager.applicationsmanager; +import java.io.IOException; import java.util.ArrayList; +import java.util.Collection; import java.util.Collections; import java.util.HashMap; import java.util.List; @@ -55,19 +57,51 @@ import org.apache.hadoop.yarn.server.resourcemanager.rmapp.attempt.RMAppAttemptState; import org.apache.hadoop.yarn.server.resourcemanager.rmcontainer.RMContainerState; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.AbstractYarnScheduler; +import org.apache.hadoop.yarn.server.resourcemanager.scheduler.ResourceScheduler; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.SchedulerApplicationAttempt; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.TestSchedulerUtils; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.YarnScheduler; +import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacityScheduler; +import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.FairScheduler; +import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fifo.FifoScheduler; import org.apache.hadoop.yarn.util.ControlledClock; import org.apache.hadoop.yarn.util.Records; import org.junit.Assert; +import org.junit.Before; import org.junit.Test; +import org.junit.runner.RunWith; +import org.junit.runners.Parameterized; +/** + * Test AM restart functions + */ +@RunWith(Parameterized.class) public class TestAMRestart { + @Parameterized.Parameters + public static Collection data() { + Collection params = new ArrayList(); + params.add(new Object[]{CapacityScheduler.class}); + params.add(new Object[]{FairScheduler.class}); + return params; + } + + private YarnConfiguration conf; + private Class schedulerConf; + + public TestAMRestart(Class schedulerParameter) { + schedulerConf = schedulerParameter; + } + + @Before + public void setup() throws IOException { + conf = new YarnConfiguration(); + conf.setClass(YarnConfiguration.RM_SCHEDULER, + schedulerConf, ResourceScheduler.class); + } + @Test(timeout = 30000) public void testAMRestartWithExistingContainers() throws Exception { - YarnConfiguration conf = new YarnConfiguration(); conf.setInt(YarnConfiguration.RM_AM_MAX_ATTEMPTS, 2); MockRM rm1 = new MockRM(conf); @@ -266,7 +300,6 @@ @Test(timeout = 30000) public void testNMTokensRebindOnAMRestart() throws Exception { - YarnConfiguration conf = new YarnConfiguration(); conf.setInt(YarnConfiguration.RM_AM_MAX_ATTEMPTS, 3); // To prevent test from blacklisting nm1 for AM, we sit threshold to half // of 2 nodes which is 1 @@ -378,7 +411,6 @@ // should not be counted towards AM max retry count. @Test(timeout = 100000) public void testShouldNotCountFailureToMaxAttemptRetry() throws Exception { - YarnConfiguration conf = new YarnConfiguration(); conf.setInt(YarnConfiguration.RM_AM_MAX_ATTEMPTS, 2); conf.setBoolean(YarnConfiguration.RECOVERY_ENABLED, true); conf.set(YarnConfiguration.RM_STORE, MemoryRMStateStore.class.getName()); @@ -503,7 +535,6 @@ @Test(timeout = 100000) public void testMaxAttemptOneMeansOne() throws Exception { - YarnConfiguration conf = new YarnConfiguration(); conf.setInt(YarnConfiguration.RM_AM_MAX_ATTEMPTS, 1); conf.setBoolean(YarnConfiguration.RECOVERY_ENABLED, true); conf.set(YarnConfiguration.RM_STORE, MemoryRMStateStore.class.getName()); @@ -537,7 +568,6 @@ // re-launch the AM. @Test(timeout = 60000) public void testPreemptedAMRestartOnRMRestart() throws Exception { - YarnConfiguration conf = new YarnConfiguration(); conf.setBoolean(YarnConfiguration.RECOVERY_ENABLED, true); conf.setBoolean(YarnConfiguration.RM_WORK_PRESERVING_RECOVERY_ENABLED, false); @@ -584,10 +614,18 @@ ApplicationStateData appState = memStore.getState().getApplicationState().get(app1.getApplicationId()); Assert.assertEquals(2, appState.getAttemptCount()); - // attempt stored has the preempted container exit status. - Assert.assertEquals(ContainerExitStatus.PREEMPTED, - appState.getAttempt(am2.getApplicationAttemptId()) - .getAMContainerExitStatus()); + if (scheduler instanceof FairScheduler || + scheduler instanceof FifoScheduler) { + // attempt stored has the preempted container exit status. + Assert.assertEquals(ContainerExitStatus.KILLED_BY_RESOURCEMANAGER, + appState.getAttempt(am2.getApplicationAttemptId()) + .getAMContainerExitStatus()); + } else { + // attempt stored has the preempted container exit status. + Assert.assertEquals(ContainerExitStatus.PREEMPTED, + appState.getAttempt(am2.getApplicationAttemptId()) + .getAMContainerExitStatus()); + } // Restart rm. MockRM rm2 = new MockRM(conf, memStore); nm1.setResourceTrackerService(rm2.getResourceTrackerService()); @@ -615,7 +653,6 @@ @Test(timeout = 50000) public void testRMRestartOrFailoverNotCountedForAMFailures() throws Exception { - YarnConfiguration conf = new YarnConfiguration(); conf.setBoolean(YarnConfiguration.RECOVERY_ENABLED, true); conf.setBoolean(YarnConfiguration.RM_WORK_PRESERVING_RECOVERY_ENABLED, false); @@ -688,7 +725,6 @@ @Test (timeout = 120000) public void testRMAppAttemptFailuresValidityInterval() throws Exception { - YarnConfiguration conf = new YarnConfiguration(); conf.setBoolean(YarnConfiguration.RECOVERY_ENABLED, true); conf.setBoolean(YarnConfiguration.RM_WORK_PRESERVING_RECOVERY_ENABLED, false); @@ -834,9 +870,8 @@ return false; } - @Test(timeout = 30000) + @Test(timeout = 40000) public void testAMRestartNotLostContainerCompleteMsg() throws Exception { - YarnConfiguration conf = new YarnConfiguration(); conf.setInt(YarnConfiguration.RM_AM_MAX_ATTEMPTS, 2); MockRM rm1 = new MockRM(conf); @@ -934,7 +969,6 @@ @Test (timeout = 20000) public void testAMRestartNotLostContainerAfterAttemptFailuresValidityInterval() throws Exception { - YarnConfiguration conf = new YarnConfiguration(); // explicitly set max-am-retry count as 2. conf.setInt(YarnConfiguration.RM_AM_MAX_ATTEMPTS, 2); @@ -1019,7 +1053,6 @@ @Test(timeout = 200000) public void testContainersFromPreviousAttemptsWithRMRestart() throws Exception { - YarnConfiguration conf = new YarnConfiguration(); conf.setInt(YarnConfiguration.RM_AM_MAX_ATTEMPTS, 2); conf.setBoolean(YarnConfiguration.RECOVERY_ENABLED, true); conf.setBoolean(