diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/api/impl/TestAMRMClient.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/api/impl/TestAMRMClient.java index 8772fdc..5a792fb 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/api/impl/TestAMRMClient.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/api/impl/TestAMRMClient.java @@ -111,8 +111,8 @@ static ApplicationAttemptId attemptId = null; static int nodeCount = 3; - static final int rolling_interval_sec = 13; - static final long am_expire_ms = 4000; + static final int ROLLING_INTERVAL_SEC = 13; + static final long AM_EXPIRE_MS = 4000; static Resource capability; static Priority priority; @@ -121,20 +121,21 @@ static String rack; static String[] nodes; static String[] racks; - private final static int DEFAULT_ITERATION = 3; + static final int DEFAULT_ITERATION = 3; - @BeforeClass - public static void setup() throws Exception { - // start minicluster + public static void setupConfiguration() { conf = new YarnConfiguration(); conf.setLong( - YarnConfiguration.RM_AMRM_TOKEN_MASTER_KEY_ROLLING_INTERVAL_SECS, - rolling_interval_sec); - conf.setLong(YarnConfiguration.RM_AM_EXPIRY_INTERVAL_MS, am_expire_ms); + YarnConfiguration.RM_AMRM_TOKEN_MASTER_KEY_ROLLING_INTERVAL_SECS, + ROLLING_INTERVAL_SEC); + conf.setLong(YarnConfiguration.RM_AM_EXPIRY_INTERVAL_MS, AM_EXPIRE_MS); conf.setInt(YarnConfiguration.RM_NM_HEARTBEAT_INTERVAL_MS, 100); // set the minimum allocation so that resource decrease can go under 1024 conf.setInt(YarnConfiguration.RM_SCHEDULER_MINIMUM_ALLOCATION_MB, 512); conf.setLong(YarnConfiguration.NM_LOG_RETAIN_SECONDS, 1); + } + + public static void setupCluster() throws Exception { yarnCluster = new MiniYARNCluster(TestAMRMClient.class.getName(), nodeCount, 1, 1); yarnCluster.init(conf); yarnCluster.start(); @@ -146,7 +147,7 @@ public static void setup() throws Exception { // get node info assertTrue("All node managers did not connect to the RM within the " - + "allotted 5-second timeout", + + "allotted 5-second timeout", yarnCluster.waitForNodeManagersToConnect(5000L)); nodeReports = yarnClient.getNodeReports(NodeState.RUNNING); assertEquals("Not all node managers were reported running", @@ -158,10 +159,16 @@ public static void setup() throws Exception { node = nodeReports.get(0).getNodeId().getHost(); rack = nodeReports.get(0).getRackName(); - nodes = new String[]{ node }; - racks = new String[]{ rack }; + nodes = new String[]{node}; + racks = new String[]{rack}; } - + + @BeforeClass + public static void setup() throws Exception { + setupConfiguration(); + setupCluster(); + } + @Before public void startApp() throws Exception { // submit new app @@ -1371,7 +1378,7 @@ public void testWaitFor() throws InterruptedException { } } - private void sleep(int sleepTime) { + protected void sleep(int sleepTime) { try { Thread.sleep(sleepTime); } catch (InterruptedException e) { @@ -1406,7 +1413,7 @@ public void testAMRMClientOnAMRMTokenRollOver() throws YarnException, // Wait for enough time and make sure the roll_over happens // At mean time, the old AMRMToken should continue to work while (System.currentTimeMillis() - startTime < - rolling_interval_sec * 1000) { + ROLLING_INTERVAL_SEC * 1000) { amClient.allocate(0.1f); try { Thread.sleep(1000); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/api/impl/TestOpportunisticContainerAllocation.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/api/impl/TestOpportunisticContainerAllocation.java index ace145d..2872507 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/api/impl/TestOpportunisticContainerAllocation.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/api/impl/TestOpportunisticContainerAllocation.java @@ -78,137 +78,14 @@ * Class that tests the allocation of OPPORTUNISTIC containers through the * centralized ResourceManager. */ -public class TestOpportunisticContainerAllocation { - private static Configuration conf = null; - private static MiniYARNCluster yarnCluster = null; - private static YarnClient yarnClient = null; - private static List nodeReports = null; - private static ApplicationAttemptId attemptId = null; - private static int nodeCount = 3; - - private static final int ROLLING_INTERVAL_SEC = 13; - private static final long AM_EXPIRE_MS = 4000; - - private static Resource capability; - private static Priority priority; - private static Priority priority2; - private static String node; - private static String rack; - private static String[] nodes; - private static String[] racks; - private final static int DEFAULT_ITERATION = 3; +public class TestOpportunisticContainerAllocation extends TestAMRMClient { @BeforeClass public static void setup() throws Exception { - // start minicluster - conf = new YarnConfiguration(); - conf.setLong( - YarnConfiguration.RM_AMRM_TOKEN_MASTER_KEY_ROLLING_INTERVAL_SECS, - ROLLING_INTERVAL_SEC); - conf.setLong(YarnConfiguration.RM_AM_EXPIRY_INTERVAL_MS, AM_EXPIRE_MS); - conf.setInt(YarnConfiguration.RM_NM_HEARTBEAT_INTERVAL_MS, 100); - // set the minimum allocation so that resource decrease can go under 1024 - conf.setInt(YarnConfiguration.RM_SCHEDULER_MINIMUM_ALLOCATION_MB, 512); + setupConfiguration(); conf.setBoolean( YarnConfiguration.OPPORTUNISTIC_CONTAINER_ALLOCATION_ENABLED, true); - conf.setLong(YarnConfiguration.NM_LOG_RETAIN_SECONDS, 1); - yarnCluster = - new MiniYARNCluster(TestAMRMClient.class.getName(), nodeCount, 1, 1); - yarnCluster.init(conf); - yarnCluster.start(); - - // start rm client - yarnClient = YarnClient.createYarnClient(); - yarnClient.init(conf); - yarnClient.start(); - - // get node info - nodeReports = yarnClient.getNodeReports(NodeState.RUNNING); - - priority = Priority.newInstance(1); - priority2 = Priority.newInstance(2); - capability = Resource.newInstance(1024, 1); - - node = nodeReports.get(0).getNodeId().getHost(); - rack = nodeReports.get(0).getRackName(); - nodes = new String[]{node}; - racks = new String[]{rack}; - } - - @Before - public void startApp() throws Exception { - // submit new app - ApplicationSubmissionContext appContext = - yarnClient.createApplication().getApplicationSubmissionContext(); - ApplicationId appId = appContext.getApplicationId(); - // set the application name - appContext.setApplicationName("Test"); - // Set the priority for the application master - Priority pri = Records.newRecord(Priority.class); - pri.setPriority(0); - appContext.setPriority(pri); - // Set the queue to which this application is to be submitted in the RM - appContext.setQueue("default"); - // Set up the container launch context for the application master - ContainerLaunchContext amContainer = BuilderUtils.newContainerLaunchContext( - Collections.emptyMap(), - new HashMap(), Arrays.asList("sleep", "100"), - new HashMap(), null, - new HashMap()); - appContext.setAMContainerSpec(amContainer); - appContext.setResource(Resource.newInstance(1024, 1)); - // Create the request to send to the applications manager - SubmitApplicationRequest appRequest = - Records.newRecord(SubmitApplicationRequest.class); - appRequest.setApplicationSubmissionContext(appContext); - // Submit the application to the applications manager - yarnClient.submitApplication(appContext); - - // wait for app to start - RMAppAttempt appAttempt = null; - while (true) { - ApplicationReport appReport = yarnClient.getApplicationReport(appId); - if (appReport.getYarnApplicationState() == - YarnApplicationState.ACCEPTED) { - attemptId = appReport.getCurrentApplicationAttemptId(); - appAttempt = yarnCluster.getResourceManager().getRMContext().getRMApps() - .get(attemptId.getApplicationId()).getCurrentAppAttempt(); - while (true) { - if (appAttempt.getAppAttemptState() == RMAppAttemptState.LAUNCHED) { - break; - } - } - break; - } - } - // Just dig into the ResourceManager and get the AMRMToken just for the sake - // of testing. - UserGroupInformation.setLoginUser(UserGroupInformation - .createRemoteUser(UserGroupInformation.getCurrentUser().getUserName())); - - // emulate RM setup of AMRM token in credentials by adding the token - // *before* setting the token service - UserGroupInformation.getCurrentUser().addToken(appAttempt.getAMRMToken()); - appAttempt.getAMRMToken() - .setService(ClientRMProxy.getAMRMTokenService(conf)); - } - - @After - public void cancelApp() throws YarnException, IOException { - yarnClient.killApplication(attemptId.getApplicationId()); - attemptId = null; - } - - @AfterClass - public static void tearDown() { - if (yarnClient != null && - yarnClient.getServiceState() == Service.STATE.STARTED) { - yarnClient.stop(); - } - if (yarnCluster != null && - yarnCluster.getServiceState() == Service.STATE.STARTED) { - yarnCluster.stop(); - } + setupCluster(); } @Test(timeout = 60000) @@ -456,12 +333,4 @@ private void testOpportunisticAllocation( assertEquals(1, receivedNMTokens.values().size()); } - - private void sleep(int sleepTime) { - try { - Thread.sleep(sleepTime); - } catch (InterruptedException e) { - e.printStackTrace(); - } - } }