diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/api/impl/TestYarnClient.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/api/impl/TestYarnClient.java index 41ef404be6b..cd0e4728421 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/api/impl/TestYarnClient.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/api/impl/TestYarnClient.java @@ -48,6 +48,7 @@ import org.apache.hadoop.io.DataInputByteBuffer; import org.apache.hadoop.io.DataOutputBuffer; import org.apache.hadoop.io.Text; +import org.apache.hadoop.metrics2.lib.DefaultMetricsSystem; import org.apache.hadoop.security.Credentials; import org.apache.hadoop.security.SecurityUtil; import org.apache.hadoop.security.UserGroupInformation; @@ -121,6 +122,7 @@ import org.apache.hadoop.yarn.server.resourcemanager.ResourceManager; import org.apache.hadoop.yarn.server.resourcemanager.reservation.ReservationSystemTestUtil; import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMApp; +import org.apache.hadoop.yarn.server.resourcemanager.scheduler.QueueMetrics; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.ResourceScheduler; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacityScheduler; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacitySchedulerConfiguration; @@ -131,14 +133,16 @@ import org.apache.log4j.LogManager; import org.apache.log4j.Logger; import org.junit.Assert; +import org.junit.Before; import org.junit.Test; import org.mockito.ArgumentCaptor; public class TestYarnClient { - @Test - public void test() { - // More to come later. + @Before + public void setup() { + QueueMetrics.clearQueueMetrics(); + DefaultMetricsSystem.setMiniClusterMode(true); } @Test diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/AbstractYarnScheduler.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/AbstractYarnScheduler.java index 2c270174845..4956b78d4f9 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/AbstractYarnScheduler.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/AbstractYarnScheduler.java @@ -1283,4 +1283,12 @@ public void asyncContainerRelease(RMContainer container) { this.rmContext.getDispatcher().getEventHandler() .handle(new ReleaseContainerEvent(container)); } + + /** + * Update internal state of the scheduler. + */ + @VisibleForTesting + public void update() { + + } } diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FairScheduler.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FairScheduler.java index c5212501b34..f4a2f3067c5 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FairScheduler.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FairScheduler.java @@ -367,6 +367,7 @@ private void dumpSchedulerState() { * required resources per job. */ @VisibleForTesting + @Override public void update() { FSQueue rootQueue = queueMgr.getRootQueue(); diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/MockRM.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/MockRM.java index 1235774d934..14c5641a005 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/MockRM.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/MockRM.java @@ -811,6 +811,8 @@ public SubmitApplicationResponse run() throws IOException, YarnException { RMAppAttemptState.SCHEDULED); } + ((AbstractYarnScheduler)getResourceScheduler()).update(); + return rmApp; } @@ -934,6 +936,7 @@ public FailApplicationAttemptResponse failApplicationAttempt( public MockAM sendAMLaunched(ApplicationAttemptId appAttemptId) throws Exception { MockAM am = new MockAM(getRMContext(), masterService, appAttemptId); + ((AbstractYarnScheduler)scheduler).update(); waitForState(appAttemptId, RMAppAttemptState.ALLOCATED); //create and set AMRMToken Token amrmToken = @@ -1158,6 +1161,7 @@ public static MockAM launchAM(RMApp app, MockRM rm, MockNM nm) RMAppAttempt attempt = waitForAttemptScheduled(app, rm); LOG.info("Launch AM " + attempt.getAppAttemptId()); nm.nodeHeartbeat(true); + ((AbstractYarnScheduler)rm.getResourceScheduler()).update(); rm.drainEventsImplicitly(); MockAM am = rm.sendAMLaunched(attempt.getAppAttemptId()); rm.waitForState(attempt.getAppAttemptId(), RMAppAttemptState.LAUNCHED); @@ -1173,6 +1177,7 @@ public static MockAM launchUAM(RMApp app, MockRM rm, MockNM nm) waitForSchedulerAppAttemptAdded(attempt.getAppAttemptId(), rm); LOG.info("Launch AM " + attempt.getAppAttemptId()); nm.nodeHeartbeat(true); + ((AbstractYarnScheduler)rm.getResourceScheduler()).update(); rm.drainEventsImplicitly(); MockAM am = new MockAM(rm.getRMContext(), rm.masterService, attempt.getAppAttemptId()); diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/ParameterizedSchedulerTestBase.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/ParameterizedSchedulerTestBase.java index 00809f04f81..289ff1cebe2 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/ParameterizedSchedulerTestBase.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/ParameterizedSchedulerTestBase.java @@ -18,53 +18,74 @@ package org.apache.hadoop.yarn.server.resourcemanager; +import org.apache.hadoop.metrics2.lib.DefaultMetricsSystem; import org.apache.hadoop.yarn.conf.YarnConfiguration; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.AbstractYarnScheduler; +import org.apache.hadoop.yarn.server.resourcemanager.scheduler.QueueMetrics; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacityScheduler; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.FairScheduler; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.FairSchedulerConfiguration; -import org.junit.Before; +import org.junit.runner.RunWith; +import org.junit.runners.Parameterized; import java.io.File; import java.io.FileWriter; import java.io.IOException; import java.io.PrintWriter; +import java.util.Arrays; +import java.util.Collection; +import java.util.stream.Collectors; +@RunWith(Parameterized.class) public abstract class ParameterizedSchedulerTestBase { protected final static String TEST_DIR = new File(System.getProperty("test.build.data", "/tmp")).getAbsolutePath(); private final static String FS_ALLOC_FILE = new File(TEST_DIR, "test-fs-queues.xml").getAbsolutePath(); - private SchedulerType schedulerType; - private YarnConfiguration conf = null; - private AbstractYarnScheduler scheduler = null; - public enum SchedulerType { CAPACITY, FAIR } + @Parameterized.Parameters(name = "{0}") + public static Collection getParameters() { + return Arrays.stream(SchedulerType.values()).map( + type -> new Object[]{type}).collect(Collectors.toList()); + } + + private SchedulerType schedulerType; + private YarnConfiguration conf = null; + private AbstractYarnScheduler scheduler = null; + public YarnConfiguration getConf() { return conf; } - @Before - public void configureScheduler() throws IOException, ClassNotFoundException { + // Due to parameterization, this gets called before each test method + public ParameterizedSchedulerTestBase(SchedulerType type) + throws IOException { conf = new YarnConfiguration(); - Class schedulerClass = - conf.getClass(YarnConfiguration.RM_SCHEDULER, - Class.forName(YarnConfiguration.DEFAULT_RM_SCHEDULER)); - - if (schedulerClass == FairScheduler.class) { - schedulerType = SchedulerType.FAIR; - configureFairScheduler(conf); - scheduler = new FairScheduler(); - } else if (schedulerClass == CapacityScheduler.class) { - schedulerType = SchedulerType.CAPACITY; - scheduler = new CapacityScheduler(); - ((CapacityScheduler)scheduler).setConf(conf); + QueueMetrics.clearQueueMetrics(); + DefaultMetricsSystem.setMiniClusterMode(true); + + schedulerType = type; + switch (schedulerType) { + case FAIR: + configureFairScheduler(conf); + scheduler = new FairScheduler(); + conf.set(YarnConfiguration.RM_SCHEDULER, + FairScheduler.class.getName()); + break; + case CAPACITY: + scheduler = new CapacityScheduler(); + ((CapacityScheduler)scheduler).setConf(conf); + conf.set(YarnConfiguration.RM_SCHEDULER, + CapacityScheduler.class.getName()); + break; + default: + throw new IllegalArgumentException("Invalid type: " + type); } } @@ -85,7 +106,6 @@ private void configureFairScheduler(YarnConfiguration conf) throws IOException { out.println(""); out.close(); - conf.set(YarnConfiguration.RM_SCHEDULER, FairScheduler.class.getName()); conf.set(FairSchedulerConfiguration.ALLOCATION_FILE, FS_ALLOC_FILE); conf.setLong(FairSchedulerConfiguration.UPDATE_INTERVAL_MS, 10); } @@ -97,7 +117,8 @@ public SchedulerType getSchedulerType() { /** * Return a scheduler configured by {@code YarnConfiguration.RM_SCHEDULER} * - *

The scheduler is configured by {@link #configureScheduler()}. + *

The scheduler is configured by + * {@link #ParameterizedSchedulerTestBase(SchedulerType)}. * Client test code can obtain the scheduler with this getter method. * Schedulers supported by this class are {@link FairScheduler} or * {@link CapacityScheduler}.

diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/RMHATestBase.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/RMHATestBase.java index 4d8b20d69fd..4ac4fc306b5 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/RMHATestBase.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/RMHATestBase.java @@ -36,6 +36,7 @@ import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMAppState; import org.apache.hadoop.yarn.server.resourcemanager.rmapp.attempt.RMAppAttempt; import org.apache.hadoop.yarn.server.resourcemanager.rmapp.attempt.RMAppAttemptState; +import org.apache.hadoop.yarn.server.resourcemanager.scheduler.AbstractYarnScheduler; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.YarnScheduler; import org.apache.hadoop.yarn.server.security.ApplicationACLsManager; import org.junit.After; @@ -95,6 +96,7 @@ protected MockAM launchAM(RMApp app, MockRM rm, MockNM nm) throws Exception { RMAppAttempt attempt = app.getCurrentAppAttempt(); nm.nodeHeartbeat(true); + ((AbstractYarnScheduler)rm.getResourceScheduler()).update(); MockAM am = rm.sendAMLaunched(attempt.getAppAttemptId()); am.registerAppAttempt(); rm.waitForState(app.getApplicationId(), RMAppState.RUNNING); diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestNodeBlacklistingOnAMFailures.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestNodeBlacklistingOnAMFailures.java index 526621004cb..e7d666ad8ad 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestNodeBlacklistingOnAMFailures.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestNodeBlacklistingOnAMFailures.java @@ -21,6 +21,7 @@ import java.util.ArrayList; import java.util.List; +import org.apache.hadoop.metrics2.lib.DefaultMetricsSystem; import org.apache.hadoop.yarn.api.records.ApplicationAttemptId; import org.apache.hadoop.yarn.api.records.ApplicationId; import org.apache.hadoop.yarn.api.records.Container; @@ -41,11 +42,14 @@ import org.apache.hadoop.yarn.server.resourcemanager.rmapp.attempt.RMAppAttemptState; import org.apache.hadoop.yarn.server.resourcemanager.rmcontainer.RMContainer; import org.apache.hadoop.yarn.server.resourcemanager.rmcontainer.RMContainerState; +import org.apache.hadoop.yarn.server.resourcemanager.scheduler.AbstractYarnScheduler; +import org.apache.hadoop.yarn.server.resourcemanager.scheduler.QueueMetrics; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.ResourceScheduler; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacityScheduler; import org.apache.hadoop.yarn.server.utils.BuilderUtils; import org.apache.hadoop.yarn.util.resource.Resources; import org.junit.Assert; +import org.junit.Before; import org.junit.Test; /** @@ -54,6 +58,12 @@ */ public class TestNodeBlacklistingOnAMFailures { + @Before + public void setup() { + QueueMetrics.clearQueueMetrics(); + DefaultMetricsSystem.setMiniClusterMode(true); + } + @Test(timeout = 100000) public void testNodeBlacklistingOnAMFailure() throws Exception { @@ -361,6 +371,7 @@ public void testNoBlacklistingForNonSystemErrors() throws Exception { // Now the AM container should be allocated RMAppAttempt attempt = MockRM.waitForAttemptScheduled(app, rm); node.nodeHeartbeat(true); + ((AbstractYarnScheduler)rm.getResourceScheduler()).update(); rm.drainEvents(); MockRM.waitForState(attempt, RMAppAttemptState.ALLOCATED, 20000); rm.sendAMLaunched(attempt.getAppAttemptId()); @@ -388,6 +399,7 @@ public void testNoBlacklistingForNonSystemErrors() throws Exception { .println("New AppAttempt launched " + attempt.getAppAttemptId()); node.nodeHeartbeat(true); + ((AbstractYarnScheduler)rm.getResourceScheduler()).update(); rm.drainEvents(); MockRM.waitForState(attempt, RMAppAttemptState.ALLOCATED, 20000); diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestRM.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestRM.java index 39313d06bd4..f912f68e910 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestRM.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestRM.java @@ -21,11 +21,13 @@ import com.google.common.base.Supplier; import org.apache.hadoop.test.GenericTestUtils; import org.apache.hadoop.yarn.event.DrainDispatcher; +import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.FairSchedulerConfiguration; import org.junit.Before; import static org.mockito.Matchers.argThat; import static org.mockito.Mockito.doNothing; import static org.mockito.Mockito.spy; +import java.io.IOException; import java.util.ArrayList; import java.util.EnumSet; import java.util.HashMap; @@ -89,6 +91,10 @@ private YarnConfiguration conf; + public TestRM(SchedulerType type) throws IOException { + super(type); + } + @Before public void setup() { conf = getConf(); diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestRMAdminService.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestRMAdminService.java index 512c14a84df..588f16deefc 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestRMAdminService.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestRMAdminService.java @@ -41,6 +41,7 @@ import org.apache.hadoop.ha.HAServiceProtocol; import org.apache.hadoop.ha.HAServiceProtocol.HAServiceState; import org.apache.hadoop.ha.HAServiceProtocol.StateChangeRequestInfo; +import org.apache.hadoop.metrics2.lib.DefaultMetricsSystem; import org.apache.hadoop.security.AccessControlException; import org.apache.hadoop.security.GroupMappingServiceProvider; import org.apache.hadoop.security.Groups; @@ -74,6 +75,7 @@ import org.apache.hadoop.yarn.server.resourcemanager.resource.DynamicResourceConfiguration; import org.apache.hadoop.yarn.server.resourcemanager.rmnode.RMNode; import org.apache.hadoop.yarn.server.resourcemanager.rmnode.RMNodeImpl; +import org.apache.hadoop.yarn.server.resourcemanager.scheduler.QueueMetrics; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacityScheduler; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacitySchedulerConfiguration; import static org.apache.hadoop.yarn.conf.YarnConfiguration.RM_PROXY_USER_PREFIX; @@ -109,6 +111,9 @@ @Before public void setup() throws IOException { + QueueMetrics.clearQueueMetrics(); + DefaultMetricsSystem.setMiniClusterMode(true); + configuration = new YarnConfiguration(); configuration.set(YarnConfiguration.RM_SCHEDULER, CapacityScheduler.class.getCanonicalName()); diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestRMRestart.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestRMRestart.java index 5cbcdbcb80d..9741d1c0cc0 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestRMRestart.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestRMRestart.java @@ -146,6 +146,10 @@ private static InetSocketAddress rmAddr; private List rms = new ArrayList(); + public TestRMRestart(SchedulerType type) throws IOException { + super(type); + } + @Before public void setup() throws IOException { conf = getConf(); @@ -383,6 +387,7 @@ public void testRMRestart() throws Exception { // assert app1 attempt is saved attempt1 = loadedApp1.getCurrentAppAttempt(); attemptId1 = attempt1.getAppAttemptId(); + ((AbstractYarnScheduler)rm2.getResourceScheduler()).update(); rm2.waitForState(attemptId1, RMAppAttemptState.ALLOCATED); appState = rmAppState.get(loadedApp1.getApplicationId()); attemptState = appState.getAttempt(attemptId1); diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestWorkPreservingRMRestart.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestWorkPreservingRMRestart.java index 2c37f44e416..a13cae719d6 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestWorkPreservingRMRestart.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestWorkPreservingRMRestart.java @@ -107,6 +107,10 @@ MockRM rm1 = null; MockRM rm2 = null; + public TestWorkPreservingRMRestart(SchedulerType type) throws IOException { + super(type); + } + @Before public void setup() throws UnknownHostException { Logger rootLogger = LogManager.getRootLogger(); diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/reservation/TestReservationSystem.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/reservation/TestReservationSystem.java index a7b7e32ff9c..6c4fac8e6cf 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/reservation/TestReservationSystem.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/reservation/TestReservationSystem.java @@ -53,6 +53,10 @@ private Configuration conf; private RMContext mockRMContext; + public TestReservationSystem(SchedulerType type) throws IOException { + super(type); + } + @Before public void setUp() throws IOException { scheduler = initializeScheduler(); diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/resourcetracker/TestNMReconnect.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/resourcetracker/TestNMReconnect.java index 6a7325c25c8..3c4e6b424de 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/resourcetracker/TestNMReconnect.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/resourcetracker/TestNMReconnect.java @@ -18,6 +18,7 @@ package org.apache.hadoop.yarn.server.resourcemanager.resourcetracker; +import java.io.IOException; import java.util.ArrayList; import java.util.List; import org.apache.hadoop.conf.Configuration; @@ -70,6 +71,10 @@ private Dispatcher dispatcher; private RMContextImpl context; + public TestNMReconnect(SchedulerType type) throws IOException { + super(type); + } + private class TestRMNodeEventDispatcher implements EventHandler { diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/TestAbstractYarnScheduler.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/TestAbstractYarnScheduler.java index 60b9e4bc95d..3399bfbd5fa 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/TestAbstractYarnScheduler.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/TestAbstractYarnScheduler.java @@ -84,13 +84,16 @@ @SuppressWarnings("unchecked") public class TestAbstractYarnScheduler extends ParameterizedSchedulerTestBase { + public TestAbstractYarnScheduler(SchedulerType type) throws IOException { + super(type); + } + @Test public void testMaximimumAllocationMemory() throws Exception { final int node1MaxMemory = 15 * 1024; final int node2MaxMemory = 5 * 1024; final int node3MaxMemory = 6 * 1024; final int configuredMaxMemory = 10 * 1024; - configureScheduler(); YarnConfiguration conf = getConf(); conf.setInt(YarnConfiguration.RM_SCHEDULER_MAXIMUM_ALLOCATION_MB, configuredMaxMemory); @@ -177,7 +180,6 @@ public void testMaximimumAllocationVCores() throws Exception { final int node2MaxVCores = 5; final int node3MaxVCores = 6; final int configuredMaxVCores = 10; - configureScheduler(); YarnConfiguration conf = getConf(); conf.setInt(YarnConfiguration.RM_SCHEDULER_MAXIMUM_ALLOCATION_VCORES, configuredMaxVCores); @@ -381,7 +383,6 @@ public void testMaxAllocationAfterUpdateNodeResource() throws IOException { @Test(timeout = 10000) public void testReleasedContainerIfAppAttemptisNull() throws Exception { YarnConfiguration conf=getConf(); - conf.setBoolean(YarnConfiguration.RECOVERY_ENABLED, true); MockRM rm1 = new MockRM(conf); try { rm1.start(); @@ -425,7 +426,6 @@ public void testReleasedContainerIfAppAttemptisNull() throws Exception { @Test(timeout=60000) public void testContainerReleasedByNode() throws Exception { System.out.println("Starting testContainerReleasedByNode"); - configureScheduler(); YarnConfiguration conf = getConf(); MockRM rm1 = new MockRM(conf); try { @@ -538,7 +538,6 @@ public void testContainerReleasedByNode() throws Exception { @Test(timeout = 60000) public void testResourceRequestRestoreWhenRMContainerIsAtAllocated() throws Exception { - configureScheduler(); YarnConfiguration conf = getConf(); MockRM rm1 = new MockRM(conf); try { @@ -627,7 +626,6 @@ public void testResourceRequestRestoreWhenRMContainerIsAtAllocated() public void testResourceRequestRecoveryToTheRightAppAttempt() throws Exception { - configureScheduler(); YarnConfiguration conf = getConf(); MockRM rm = new MockRM(conf); try { @@ -798,7 +796,6 @@ private ResourceTrackerService getPrivateResourceTrackerService( */ @Test(timeout = 60000) public void testNodemanagerReconnect() throws Exception { - configureScheduler(); Configuration conf = getConf(); MockRM rm = new MockRM(conf); try { diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/TestSchedulingWithAllocationRequestId.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/TestSchedulingWithAllocationRequestId.java index e60fd6f889a..11a6ef14e18 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/TestSchedulingWithAllocationRequestId.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/TestSchedulingWithAllocationRequestId.java @@ -18,6 +18,7 @@ package org.apache.hadoop.yarn.server.resourcemanager.scheduler; +import java.io.IOException; import java.util.List; import org.apache.hadoop.yarn.api.protocolrecords.AllocateResponse; @@ -30,6 +31,7 @@ import org.apache.hadoop.yarn.server.resourcemanager.ParameterizedSchedulerTestBase; import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMApp; import org.apache.hadoop.yarn.server.resourcemanager.rmapp.attempt.RMAppAttempt; +import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.FairSchedulerConfiguration; import org.junit.Assert; import org.junit.Test; import org.slf4j.Logger; @@ -46,9 +48,21 @@ LoggerFactory.getLogger(TestSchedulingWithAllocationRequestId.class); private static final int GB = 1024; - @Test + public TestSchedulingWithAllocationRequestId(SchedulerType type) throws IOException { + super(type); + } + + @Override + public YarnConfiguration getConf() { + YarnConfiguration conf = super.getConf(); + if (getSchedulerType().equals(SchedulerType.FAIR)) { + conf.setBoolean(FairSchedulerConfiguration.ASSIGN_MULTIPLE, true); + } + return conf; + } + + @Test (timeout = 10000) public void testMultipleAllocationRequestIds() throws Exception { - configureScheduler(); YarnConfiguration conf = getConf(); MockRM rm = new MockRM(conf); try { @@ -63,32 +77,20 @@ public void testMultipleAllocationRequestIds() throws Exception { MockAM am1 = rm.sendAMLaunched(attempt1.getAppAttemptId()); am1.registerAppAttempt(); - // add request for containers with id 10 & 20 - am1.addRequests(new String[] {"127.0.0.1" }, 2 * GB, 1, 1, 10L); - AllocateResponse allocResponse = am1.schedule(); // send the request - am1.addRequests(new String[] {"127.0.0.2" }, 2 * GB, 1, 2, 20L); - allocResponse = am1.schedule(); // send the request + // send requests for containers with id 10 & 20 + am1.allocate(am1.createReq( + new String[] {"127.0.0.1"}, 2 * GB, 1, 1, 10L), null); + am1.allocate(am1.createReq( + new String[] {"127.0.0.2"}, 2 * GB, 1, 2, 20L), null); // check if request id 10 is satisfied - nm1.nodeHeartbeat(true); - allocResponse = am1.schedule(); // send the request - while (allocResponse.getAllocatedContainers().size() < 1) { - LOG.info("Waiting for containers to be created for app 1..."); - Thread.sleep(100); - allocResponse = am1.schedule(); - } + AllocateResponse allocResponse = waitForAllocResponse(rm, am1, nm1, 1); List allocated = allocResponse.getAllocatedContainers(); Assert.assertEquals(1, allocated.size()); checkAllocatedContainer(allocated.get(0), 2 * GB, nm1.getNodeId(), 10); // check now if request id 20 is satisfied - nm2.nodeHeartbeat(true); - while (allocResponse.getAllocatedContainers().size() < 2) { - LOG.info("Waiting for containers to be created for app 1..."); - Thread.sleep(100); - allocResponse = am1.schedule(); - } - + allocResponse = waitForAllocResponse(rm, am1, nm2, 2); allocated = allocResponse.getAllocatedContainers(); Assert.assertEquals(2, allocated.size()); for (Container container : allocated) { @@ -101,9 +103,8 @@ public void testMultipleAllocationRequestIds() throws Exception { } } - @Test + @Test (timeout = 10000) public void testMultipleAllocationRequestDiffPriority() throws Exception { - configureScheduler(); YarnConfiguration conf = getConf(); MockRM rm = new MockRM(conf); try { @@ -118,20 +119,14 @@ public void testMultipleAllocationRequestDiffPriority() throws Exception { MockAM am1 = rm.sendAMLaunched(attempt1.getAppAttemptId()); am1.registerAppAttempt(); - // add request for containers with id 10 & 20 - am1.addRequests(new String[] {"127.0.0.1" }, 2 * GB, 2, 1, 10L); - AllocateResponse allocResponse = am1.schedule(); // send the request - am1.addRequests(new String[] {"127.0.0.2" }, 2 * GB, 1, 2, 20L); - allocResponse = am1.schedule(); // send the request + // send requests for containers with id 10 & 20 + am1.allocate(am1.createReq( + new String[] {"127.0.0.1"}, 2 * GB, 2, 1, 10L), null); + am1.allocate(am1.createReq( + new String[] {"127.0.0.2"}, 2 * GB, 1, 2, 20L), null); // check if request id 20 is satisfied first - nm2.nodeHeartbeat(true); - while (allocResponse.getAllocatedContainers().size() < 2) { - LOG.info("Waiting for containers to be created for app 1..."); - Thread.sleep(100); - allocResponse = am1.schedule(); - } - + AllocateResponse allocResponse = waitForAllocResponse(rm, am1, nm2, 2); List allocated = allocResponse.getAllocatedContainers(); Assert.assertEquals(2, allocated.size()); for (Container container : allocated) { @@ -139,13 +134,7 @@ public void testMultipleAllocationRequestDiffPriority() throws Exception { } // check now if request id 10 is satisfied - nm1.nodeHeartbeat(true); - allocResponse = am1.schedule(); // send the request - while (allocResponse.getAllocatedContainers().size() < 1) { - LOG.info("Waiting for containers to be created for app 1..."); - Thread.sleep(100); - allocResponse = am1.schedule(); - } + allocResponse = waitForAllocResponse(rm, am1, nm1, 1); allocated = allocResponse.getAllocatedContainers(); Assert.assertEquals(1, allocated.size()); checkAllocatedContainer(allocated.get(0), 2 * GB, nm1.getNodeId(), 10); @@ -164,9 +153,8 @@ private void checkAllocatedContainer(Container allocated, int memory, allocated.getAllocationRequestId()); } - @Test + @Test (timeout = 10000) public void testMultipleAppsWithAllocationReqId() throws Exception { - configureScheduler(); YarnConfiguration conf = getConf(); MockRM rm = new MockRM(conf); try { @@ -190,19 +178,11 @@ public void testMultipleAppsWithAllocationReqId() throws Exception { // Submit app1 RR with allocationReqId = 5 int numContainers = 1; - am1.addRequests(new String[] {host0, host1 }, 1 * GB, 1, numContainers, - 5L); - AllocateResponse allocResponse = am1.schedule(); - - // wait for containers to be allocated. - nm1.nodeHeartbeat(true); - allocResponse = am1.schedule(); // send the request - while (allocResponse.getAllocatedContainers().size() < 1) { - LOG.info("Waiting for containers to be created for app 1..."); - Thread.sleep(100); - allocResponse = am1.schedule(); - } + am1.allocate(am1.createReq( + new String[] {host0, host1}, 1 * GB, 1, numContainers, 5L), null); + // wait for container to be allocated. + AllocateResponse allocResponse = waitForAllocResponse(rm, am1, nm1, 1); List allocated = allocResponse.getAllocatedContainers(); Assert.assertEquals(1, allocated.size()); checkAllocatedContainer(allocated.get(0), 1 * GB, nm1.getNodeId(), 5L); @@ -212,55 +192,31 @@ public void testMultipleAppsWithAllocationReqId() throws Exception { MockAM am2 = MockRM.launchAndRegisterAM(app2, rm, nm2); // Submit app2 RR with allocationReqId = 5 - am2.addRequests(new String[] {host0, host1 }, 2 * GB, 1, numContainers, - 5L); - am2.schedule(); - - // wait for containers to be allocated. - nm2.nodeHeartbeat(true); - allocResponse = am2.schedule(); // send the request - while (allocResponse.getAllocatedContainers().size() < 1) { - LOG.info("Waiting for containers to be created for app 1..."); - Thread.sleep(100); - allocResponse = am2.schedule(); - } + am2.allocate(am1.createReq( + new String[] {host0, host1}, 2 * GB, 1, numContainers, 5L), null); + // wait for container to be allocated. + allocResponse = waitForAllocResponse(rm, am2, nm2, 1); allocated = allocResponse.getAllocatedContainers(); Assert.assertEquals(1, allocated.size()); checkAllocatedContainer(allocated.get(0), 2 * GB, nm2.getNodeId(), 5L); // Now submit app2 RR with allocationReqId = 10 - am2.addRequests(new String[] {host0, host1 }, 3 * GB, 1, numContainers, - 10L); - am2.schedule(); - - // wait for containers to be allocated. - nm1.nodeHeartbeat(true); - allocResponse = am2.schedule(); // send the request - while (allocResponse.getAllocatedContainers().size() < 1) { - LOG.info("Waiting for containers to be created for app 1..."); - Thread.sleep(100); - allocResponse = am2.schedule(); - } + am2.allocate(am1.createReq( + new String[] {host0, host1}, 3 * GB, 1, numContainers, 10L), null); + // wait for container to be allocated. + allocResponse = waitForAllocResponse(rm, am2, nm1, 1); allocated = allocResponse.getAllocatedContainers(); Assert.assertEquals(1, allocated.size()); checkAllocatedContainer(allocated.get(0), 3 * GB, nm1.getNodeId(), 10L); // Now submit app1 RR with allocationReqId = 10 - am1.addRequests(new String[] {host0, host1 }, 4 * GB, 1, numContainers, - 10L); - am1.schedule(); - - // wait for containers to be allocated. - nm2.nodeHeartbeat(true); - allocResponse = am1.schedule(); // send the request - while (allocResponse.getAllocatedContainers().size() < 1) { - LOG.info("Waiting for containers to be created for app 1..."); - Thread.sleep(100); - allocResponse = am1.schedule(); - } + am1.allocate(am1.createReq( + new String[] {host0, host1}, 4 * GB, 1, numContainers, 10L), null); + // wait for container to be allocated. + allocResponse = waitForAllocResponse(rm, am1, nm2, 1); allocated = allocResponse.getAllocatedContainers(); Assert.assertEquals(1, allocated.size()); checkAllocatedContainer(allocated.get(0), 4 * GB, nm2.getNodeId(), 10L); @@ -271,4 +227,17 @@ public void testMultipleAppsWithAllocationReqId() throws Exception { } } + private AllocateResponse waitForAllocResponse(MockRM rm, MockAM am, MockNM nm, + int size) throws Exception { + AllocateResponse allocResponse = am.doHeartbeat(); + while (allocResponse.getAllocatedContainers().size() < size) { + LOG.info("Waiting for containers to be created for app..."); + nm.nodeHeartbeat(true); + ((AbstractYarnScheduler) rm.getResourceScheduler()).update(); + Thread.sleep(100); + allocResponse = am.doHeartbeat(); + } + return allocResponse; + } + } diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/policy/TestFairOrderingPolicy.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/policy/TestFairOrderingPolicy.java index 37fc3b3474c..683173af709 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/policy/TestFairOrderingPolicy.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/policy/TestFairOrderingPolicy.java @@ -21,6 +21,7 @@ import java.util.*; import org.apache.hadoop.yarn.api.records.NodeId; +import org.apache.hadoop.yarn.conf.YarnConfiguration; import org.apache.hadoop.yarn.server.resourcemanager.MockRM; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacityScheduler; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.LeafQueue; @@ -148,7 +149,10 @@ public void testSizeBasedWeightNotAffectAppActivation() throws Exception { // Define top-level queues String queuePath = CapacitySchedulerConfiguration.ROOT + ".default"; - csConf.setOrderingPolicy(queuePath, CapacitySchedulerConfiguration.FAIR_APP_ORDERING_POLICY); + csConf.set(YarnConfiguration.RM_SCHEDULER, + CapacityScheduler.class.getCanonicalName()); + csConf.setOrderingPolicy(queuePath, + CapacitySchedulerConfiguration.FAIR_APP_ORDERING_POLICY); csConf.setOrderingPolicyParameter(queuePath, FairOrderingPolicy.ENABLE_SIZE_BASED_WEIGHT, "true"); csConf.setMaximumApplicationMasterResourcePerQueuePercent(queuePath, 0.1f); diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/security/TestClientToAMTokens.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/security/TestClientToAMTokens.java index d4e7727ad5e..a0f4007cf8a 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/security/TestClientToAMTokens.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/security/TestClientToAMTokens.java @@ -86,6 +86,10 @@ public class TestClientToAMTokens extends ParameterizedSchedulerTestBase { private YarnConfiguration conf; + public TestClientToAMTokens(SchedulerType type) throws IOException { + super(type); + } + @Before public void setup() { conf = getConf();