diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/api/impl/TestYarnClient.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/api/impl/TestYarnClient.java index 4c1a9cf..2d57417e 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/api/impl/TestYarnClient.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/api/impl/TestYarnClient.java @@ -26,7 +26,9 @@ import static org.mockito.Mockito.verify; import static org.mockito.Mockito.when; +import java.io.FileWriter; import java.io.IOException; +import java.io.PrintWriter; import java.lang.Thread.State; import java.nio.ByteBuffer; import java.security.PrivilegedExceptionAction; @@ -96,7 +98,6 @@ import org.apache.hadoop.yarn.api.records.ContainerState; import org.apache.hadoop.yarn.api.records.FinalApplicationStatus; import org.apache.hadoop.yarn.api.records.NodeId; -import org.apache.hadoop.yarn.api.records.NodeLabel; import org.apache.hadoop.yarn.api.records.Priority; import org.apache.hadoop.yarn.api.records.ReservationDefinition; import org.apache.hadoop.yarn.api.records.ReservationId; @@ -119,13 +120,15 @@ import org.apache.hadoop.yarn.security.client.TimelineDelegationTokenIdentifier; import org.apache.hadoop.yarn.server.MiniYARNCluster; import org.apache.hadoop.yarn.server.resourcemanager.MockRM; +import org.apache.hadoop.yarn.server.resourcemanager.ParameterizedSchedulerTestBase; import org.apache.hadoop.yarn.server.resourcemanager.ResourceManager; import org.apache.hadoop.yarn.server.resourcemanager.reservation.ReservationSystemTestUtil; import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMApp; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.QueueMetrics; -import org.apache.hadoop.yarn.server.resourcemanager.scheduler.ResourceScheduler; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacityScheduler; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacitySchedulerConfiguration; +import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.FairScheduler; +import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.FairSchedulerConfiguration; import org.apache.hadoop.yarn.util.Clock; import org.apache.hadoop.yarn.util.Records; import org.apache.hadoop.yarn.util.UTCClock; @@ -135,7 +138,16 @@ import org.mockito.ArgumentCaptor; import org.slf4j.event.Level; -public class TestYarnClient { +/** + * This class is to test class {@link YarnClient) and {@link YarnClientImpl}. + */ +public class TestYarnClient extends ParameterizedSchedulerTestBase { + + public TestYarnClient(SchedulerType type) throws IOException { + super(type); + } + + protected void configureFairScheduler(YarnConfiguration conf) {} @Before public void setup() { @@ -145,7 +157,7 @@ public void setup() { @Test public void testClientStop() { - Configuration conf = new Configuration(); + Configuration conf = getConf(); ResourceManager rm = new ResourceManager(); rm.init(conf); rm.start(); @@ -159,7 +171,7 @@ public void testClientStop() { @Test public void testStartWithTimelineV15() throws Exception { - Configuration conf = new Configuration(); + Configuration conf = getConf(); conf.setBoolean(YarnConfiguration.TIMELINE_SERVICE_ENABLED, true); conf.setFloat(YarnConfiguration.TIMELINE_SERVICE_VERSION, 1.5f); YarnClientImpl client = (YarnClientImpl) YarnClient.createYarnClient(); @@ -254,7 +266,7 @@ public void verifyError(Throwable e) { @SuppressWarnings("deprecation") @Test (timeout = 30000) public void testSubmitApplication() throws Exception { - Configuration conf = new Configuration(); + Configuration conf = getConf(); conf.setLong(YarnConfiguration.YARN_CLIENT_APP_SUBMISSION_POLL_INTERVAL_MS, 100); // speed up tests final YarnClient client = new MockYarnClient(); @@ -301,7 +313,7 @@ public void testSubmitApplication() throws Exception { @SuppressWarnings("deprecation") @Test (timeout = 20000) public void testSubmitApplicationInterrupted() throws IOException { - Configuration conf = new Configuration(); + Configuration conf = getConf(); int pollIntervalMs = 1000; conf.setLong(YarnConfiguration.YARN_CLIENT_APP_SUBMISSION_POLL_INTERVAL_MS, pollIntervalMs); @@ -406,10 +418,9 @@ public void testKillApplication() throws Exception { rm.start(); RMApp app = rm.submitApp(2000); - Configuration conf = new Configuration(); @SuppressWarnings("resource") final YarnClient client = new MockYarnClient(); - client.init(conf); + client.init(getConf()); client.start(); client.killApplication(app.getApplicationId()); @@ -447,9 +458,8 @@ public void testApplicationTypeLimit() throws Exception { @Test (timeout = 10000) public void testGetApplications() throws YarnException, IOException { - Configuration conf = new Configuration(); final YarnClient client = new MockYarnClient(); - client.init(conf); + client.init(getConf()); client.start(); List expectedReports = ((MockYarnClient)client).getReports(); @@ -500,9 +510,8 @@ public void testGetApplications() throws YarnException, IOException { @Test(timeout = 10000) public void testGetApplicationAttempts() throws YarnException, IOException { - Configuration conf = new Configuration(); final YarnClient client = new MockYarnClient(); - client.init(conf); + client.init(getConf()); client.start(); ApplicationId applicationId = ApplicationId.newInstance(1234, 5); @@ -539,7 +548,7 @@ public void testGetApplicationAttempt() throws YarnException, IOException { @Test(timeout = 10000) public void testGetContainers() throws YarnException, IOException { - Configuration conf = new Configuration(); + Configuration conf = getConf(); conf.setBoolean(YarnConfiguration.APPLICATION_HISTORY_ENABLED, true); @@ -572,7 +581,7 @@ public void testGetContainers() throws YarnException, IOException { @Test(timeout = 10000) public void testGetContainerReport() throws YarnException, IOException { - Configuration conf = new Configuration(); + Configuration conf = getConf(); conf.setBoolean(YarnConfiguration.APPLICATION_HISTORY_ENABLED, true); final YarnClient client = new MockYarnClient(); @@ -603,9 +612,8 @@ public void testGetContainerReport() throws YarnException, IOException { @Test (timeout = 10000) public void testGetLabelsToNodes() throws YarnException, IOException { - Configuration conf = new Configuration(); final YarnClient client = new MockYarnClient(); - client.init(conf); + client.init(getConf()); client.start(); // Get labels to nodes mapping @@ -629,9 +637,8 @@ public void testGetLabelsToNodes() throws YarnException, IOException { @Test (timeout = 10000) public void testGetNodesToLabels() throws YarnException, IOException { - Configuration conf = new Configuration(); final YarnClient client = new MockYarnClient(); - client.init(conf); + client.init(getConf()); client.start(); // Get labels to nodes mapping @@ -1025,7 +1032,7 @@ public void testAMMRTokens() throws Exception { MiniYARNCluster cluster = new MiniYARNCluster("testMRAMTokens", 1, 1, 1); YarnClient rmClient = null; try { - cluster.init(new YarnConfiguration()); + cluster.init(getConf()); cluster.start(); final Configuration yarnConf = cluster.getConfig(); rmClient = YarnClient.createYarnClient(); @@ -1146,7 +1153,7 @@ private void testAsyncAPIPollTimeoutHelper(Long valueForTimeout, boolean expectedTimeoutEnforcement) { YarnClientImpl client = new YarnClientImpl(); try { - Configuration conf = new Configuration(); + Configuration conf = getConf(); if (valueForTimeout != null) { conf.setLong( YarnConfiguration.YARN_CLIENT_APPLICATION_CLIENT_PROTOCOL_POLL_TIMEOUT_MS, @@ -1165,7 +1172,7 @@ private void testAsyncAPIPollTimeoutHelper(Long valueForTimeout, @Test public void testBestEffortTimelineDelegationToken() throws Exception { - Configuration conf = new YarnConfiguration(); + Configuration conf = getConf(); conf.setBoolean(YarnConfiguration.TIMELINE_SERVICE_ENABLED, true); SecurityUtil.setAuthenticationMethod(AuthenticationMethod.KERBEROS, conf); @@ -1199,7 +1206,7 @@ TimelineClient createTimelineClient() throws IOException, YarnException { @Test public void testAutomaticTimelineDelegationTokenLoading() throws Exception { - Configuration conf = new YarnConfiguration(); + Configuration conf = getConf(); conf.setBoolean(YarnConfiguration.TIMELINE_SERVICE_ENABLED, true); SecurityUtil.setAuthenticationMethod(AuthenticationMethod.KERBEROS, conf); TimelineDelegationTokenIdentifier timelineDT = @@ -1289,7 +1296,7 @@ public boolean isSecurityEnabled() { public void testParseTimelineDelegationTokenRenewer() throws Exception { // Client side YarnClientImpl client = (YarnClientImpl) YarnClient.createYarnClient(); - Configuration conf = new YarnConfiguration(); + Configuration conf = getConf(); conf.setBoolean(YarnConfiguration.TIMELINE_SERVICE_ENABLED, true); conf.set(YarnConfiguration.RM_PRINCIPAL, "rm/_HOST@EXAMPLE.COM"); conf.set( @@ -1304,15 +1311,10 @@ public void testParseTimelineDelegationTokenRenewer() throws Exception { } private MiniYARNCluster setupMiniYARNCluster() throws Exception { - CapacitySchedulerConfiguration conf = new CapacitySchedulerConfiguration(); - ReservationSystemTestUtil.setupQueueConfiguration(conf); - conf.setClass(YarnConfiguration.RM_SCHEDULER, CapacityScheduler.class, - ResourceScheduler.class); - conf.setBoolean(YarnConfiguration.RM_RESERVATION_SYSTEM_ENABLE, true); MiniYARNCluster cluster = new MiniYARNCluster("testReservationAPIs", 2, 1, 1); - cluster.init(conf); + cluster.init(getConfigurationForReservation()); cluster.start(); GenericTestUtils.waitFor(new Supplier() { @@ -1328,6 +1330,53 @@ public Boolean get() { return cluster; } + private Configuration getConfigurationForReservation() { + Configuration conf = new Configuration(); + if (getSchedulerType() == SchedulerType.FAIR) { + conf = configureReservationForFairScheduler(); + conf.set(YarnConfiguration.RM_SCHEDULER, FairScheduler.class.getName()); + } else if (getSchedulerType() == SchedulerType.CAPACITY) { + conf = configureReservationForCapacityScheduler(); + conf.set(YarnConfiguration.RM_SCHEDULER, + CapacityScheduler.class.getName()); + } + + conf.setBoolean(YarnConfiguration.RM_RESERVATION_SYSTEM_ENABLE, true); + return conf; + } + + private Configuration configureReservationForCapacityScheduler() { + CapacitySchedulerConfiguration conf = new CapacitySchedulerConfiguration(); + ReservationSystemTestUtil.setupQueueConfiguration(conf); + return conf; + } + + private Configuration configureReservationForFairScheduler() { + Configuration conf = new Configuration(); + try { + PrintWriter out = new PrintWriter(new FileWriter(FS_ALLOC_FILE)); + out.println(""); + out.println(""); + out.println(""); + out.println(" "); + out.println(" "); + out.println(" "); + // set weight to 10 to make sure this queue get enough steady fair share + out.println(" 10"); + out.println(" "); + out.println(""); + out.println("drf" + + ""); + out.println(""); + out.close(); + } catch (IOException e) { + Assert.fail(e.getMessage()); + } + + conf.set(FairSchedulerConfiguration.ALLOCATION_FILE, FS_ALLOC_FILE); + return conf; + } + private YarnClient setupYarnClient(MiniYARNCluster cluster) { final Configuration yarnConf = cluster.getConfig(); YarnClient client = YarnClient.createYarnClient(); @@ -1683,7 +1732,7 @@ private ReservationSubmissionRequest createSimpleReservationRequest( @Test(timeout = 30000, expected = ApplicationNotFoundException.class) public void testShouldNotRetryForeverForNonNetworkExceptions() throws Exception { - YarnConfiguration conf = new YarnConfiguration(); + YarnConfiguration conf = getConf(); conf.setInt(YarnConfiguration.RESOURCEMANAGER_CONNECT_MAX_WAIT_MS, -1); ResourceManager rm = null; @@ -1715,10 +1764,9 @@ public void testShouldNotRetryForeverForNonNetworkExceptions() throws Exception @Test public void testSignalContainer() throws Exception { - Configuration conf = new Configuration(); @SuppressWarnings("resource") final YarnClient client = new MockYarnClient(); - client.init(conf); + client.init(getConf()); client.start(); ApplicationId applicationId = ApplicationId.newInstance(1234, 5); ApplicationAttemptId appAttemptId = ApplicationAttemptId.newInstance( @@ -1741,7 +1789,7 @@ private void testCreateTimelineClientWithError( boolean timelineClientBestEffort, Throwable mockErr, CreateTimelineClientErrorVerifier errVerifier) throws Exception { - Configuration conf = new Configuration(); + Configuration conf = getConf(); conf.setBoolean(YarnConfiguration.TIMELINE_SERVICE_ENABLED, timelineServiceEnabled); conf.setBoolean(YarnConfiguration.TIMELINE_SERVICE_CLIENT_BEST_EFFORT, diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/ParameterizedSchedulerTestBase.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/ParameterizedSchedulerTestBase.java index 9a29a89..2451906 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/ParameterizedSchedulerTestBase.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/ParameterizedSchedulerTestBase.java @@ -41,7 +41,7 @@ public abstract class ParameterizedSchedulerTestBase { protected final static String TEST_DIR = new File(System.getProperty("test.build.data", "/tmp")).getAbsolutePath(); - private final static String FS_ALLOC_FILE = + protected final static String FS_ALLOC_FILE = new File(TEST_DIR, "test-fs-queues.xml").getAbsolutePath(); public enum SchedulerType {