diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/api/impl/TestYarnClientWithReservation.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/api/impl/TestYarnClientWithReservation.java index 8f1be992e45..717647724ac 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/api/impl/TestYarnClientWithReservation.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/api/impl/TestYarnClientWithReservation.java @@ -45,11 +45,6 @@ import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacitySchedulerConfiguration; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.FairScheduler; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.FairSchedulerConfiguration; - -import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair - .allocationfile.AllocationFileQueue; -import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair - .allocationfile.AllocationFileWriter; import org.apache.hadoop.yarn.util.Clock; import org.apache.hadoop.yarn.util.UTCClock; import org.junit.Assert; @@ -141,16 +136,25 @@ private Configuration configureReservationForCapacityScheduler() { private Configuration configureReservationForFairScheduler() { Configuration conf = new Configuration(); - AllocationFileWriter.create() - .defaultQueueSchedulingPolicy("drf") - .addQueue(new AllocationFileQueue.Builder("root") - .subQueue(new AllocationFileQueue.Builder("default").build()) - .subQueue(new AllocationFileQueue.Builder("dedicated") - .reservation() - .weight(10) - .build()) - .build()) - .writeToFile(FS_ALLOC_FILE); + try { + PrintWriter out = new PrintWriter(new FileWriter(FS_ALLOC_FILE)); + out.println(""); + out.println(""); + out.println(""); + out.println(" "); + out.println(" "); + out.println(" "); + // set weight to 10 to make sure this queue get enough steady fair share + out.println(" 10"); + out.println(" "); + out.println(""); + out.println("drf" + + ""); + out.println(""); + out.close(); + } catch (IOException e) { + Assert.fail(e.getMessage()); + } conf.set(FairSchedulerConfiguration.ALLOCATION_FILE, FS_ALLOC_FILE); return conf; diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/ParameterizedSchedulerTestBase.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/ParameterizedSchedulerTestBase.java index f4193fb6cc0..4de16dc88b0 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/ParameterizedSchedulerTestBase.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/ParameterizedSchedulerTestBase.java @@ -26,12 +26,6 @@ import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.FairScheduler; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.FairSchedulerConfiguration; - - -import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair - .allocationfile.AllocationFileQueue; -import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair - .allocationfile.AllocationFileWriter; import org.junit.After; import org.junit.runner.RunWith; import org.junit.runners.Parameterized; @@ -99,17 +93,20 @@ public ParameterizedSchedulerTestBase(SchedulerType type) protected void configureFairScheduler(YarnConfiguration conf) throws IOException { // Disable queueMaxAMShare limitation for fair scheduler - AllocationFileWriter.create() - .defaultQueueSchedulingPolicy("fair") - .queueMaxAMShareDefault(-1.0) - .addQueue(new AllocationFileQueue.Builder("root") - .schedulingPolicy("drf") - .weight(1.0f) - .fairSharePreemptionTimeout(100) - .minSharePreemptionTimeout(120) - .fairSharePreemptionThreshold(.5) - .build()) - .writeToFile(FS_ALLOC_FILE); + PrintWriter out = new PrintWriter(new FileWriter(FS_ALLOC_FILE)); + out.println(""); + out.println(""); + out.println("-1.0"); + out.println("fair"); + out.println(""); + out.println(" drf"); + out.println(" 1.0"); + out.println(" 100"); + out.println(" 120"); + out.println(" .5"); + out.println(""); + out.println(""); + out.close(); conf.set(FairSchedulerConfiguration.ALLOCATION_FILE, FS_ALLOC_FILE); conf.setLong(FairSchedulerConfiguration.UPDATE_INTERVAL_MS, 10); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/ReservationACLsTestBase.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/ReservationACLsTestBase.java index e9ba98ea07f..c8ee00e60bc 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/ReservationACLsTestBase.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/ReservationACLsTestBase.java @@ -53,12 +53,6 @@ import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacitySchedulerConfiguration; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.FairScheduler; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.FairSchedulerConfiguration; - - -import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair - .allocationfile.AllocationFileQueue; -import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair - .allocationfile.AllocationFileWriter; import org.apache.hadoop.yarn.server.utils.BuilderUtils; import org.junit.After; import org.junit.Assert; @@ -558,34 +552,48 @@ private static Configuration createCapacitySchedulerConfiguration() { return csConf; } - private static Configuration createFairSchedulerConfiguration() { + private static Configuration createFairSchedulerConfiguration() throws + IOException { FairSchedulerConfiguration fsConf = new FairSchedulerConfiguration(); final String TEST_DIR = new File(System.getProperty("test.build.data", "/tmp")).getAbsolutePath(); final String ALLOC_FILE = new File(TEST_DIR, "test-queues.xml") .getAbsolutePath(); - - AllocationFileWriter.create() - .defaultQueueSchedulingPolicy("drf") - .addQueue(new AllocationFileQueue.Builder("queueA") - .aclSubmitReservations("queueA_user,common_user ") - .aclAdministerReservations("queueA_admin ") - .aclListReservations("common_user ") - .aclSubmitApps("queueA_user,common_user ") - .aclAdministerApps("queueA_admin ") - .reservation().build()) - .addQueue(new AllocationFileQueue.Builder("queueB") - .aclSubmitReservations("queueB_user,common_user ") - .aclAdministerReservations("queueB_admin ") - .aclListReservations("common_user ") - .aclSubmitApps("queueB_user,common_user ") - .aclAdministerApps("queueB_admin ") - .reservation().build()) - .addQueue(new AllocationFileQueue.Builder("queueC") - .reservation().build()) - .writeToFile(ALLOC_FILE); - + PrintWriter out = new PrintWriter(new FileWriter(ALLOC_FILE)); + out.println(""); + out.println(""); + out.println(" drf" + + ""); + out.println(" "); + out.println(" " + + "queueA_user,common_user " + + ""); + out.println(" " + + "queueA_admin " + + ""); + out.println(" common_user "); + out.println(" queueA_user,common_user "); + out.println(" queueA_admin "); + out.println(" "); + out.println(" "); + out.println(" "); + out.println(" queueB_user,common_user "); + out.println(" queueB_admin "); + out.println(" " + + "queueB_user,common_user " + + ""); + out.println(" " + + "queueB_admin " + + ""); + out.println(" common_user "); + out.println(" "); + out.println(" "); + out.println(" "); + out.println(" "); + out.println(" "); + out.println(""); + out.close(); fsConf.set(FairSchedulerConfiguration.ALLOCATION_FILE, ALLOC_FILE); fsConf.setBoolean(YarnConfiguration.RM_RESERVATION_SYSTEM_ENABLE, true); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestAppManagerWithFairScheduler.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestAppManagerWithFairScheduler.java index 337c0bb21e2..d0dcae0ad4c 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestAppManagerWithFairScheduler.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestAppManagerWithFairScheduler.java @@ -47,12 +47,6 @@ import org.apache.hadoop.yarn.server.resourcemanager.scheduler.ResourceScheduler; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.FairScheduler; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.FairSchedulerConfiguration; - - -import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair - .allocationfile.AllocationFileQueue; -import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair - .allocationfile.AllocationFileWriter; import org.apache.hadoop.yarn.server.resourcemanager.security.ClientToAMTokenSecretManagerInRM; import org.apache.hadoop.yarn.server.security.ApplicationACLsManager; import org.apache.hadoop.yarn.util.Records; @@ -79,9 +73,13 @@ @Before public void setup() throws IOException { // Basic config with one queue (override in test if needed) - AllocationFileWriter.create() - .addQueue(new AllocationFileQueue.Builder("test").build()) - .writeToFile(allocFileName); + PrintWriter out = new PrintWriter(new FileWriter(allocFileName)); + out.println(""); + out.println(""); + out.println(" "); + out.println(" "); + out.println(""); + out.close(); conf.setClass(YarnConfiguration.RM_SCHEDULER, FairScheduler.class, ResourceScheduler.class); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestApplicationACLs.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestApplicationACLs.java index 4718f9698a3..e952c50353e 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestApplicationACLs.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestApplicationACLs.java @@ -35,16 +35,6 @@ import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMApp; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.FairScheduler; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.FairSchedulerConfiguration; - - -import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair - .allocationfile.AllocationFileQueue; -import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair - .allocationfile.AllocationFileQueuePlacementPolicy; -import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair - .allocationfile.AllocationFileQueuePlacementRule; -import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair - .allocationfile.AllocationFileWriter; import org.junit.After; import org.junit.Assert; import org.slf4j.Logger; @@ -191,22 +181,25 @@ public void tearDown() { } @Override - protected void configureFairScheduler(YarnConfiguration conf) { + protected void configureFairScheduler(YarnConfiguration conf) + throws IOException { final String testDir = new File(System.getProperty("test.build.data", "/tmp")).getAbsolutePath(); final String allocFile = new File(testDir, "test-queues.xml") .getAbsolutePath(); - - AllocationFileWriter.create() - .addQueue(new AllocationFileQueue.Builder("root") - .subQueue(new AllocationFileQueue.Builder("default").build()) - .build()) - .queuePlacementPolicy(new AllocationFileQueuePlacementPolicy() - .addRule(new AllocationFileQueuePlacementRule("specified") - .create(false)) - .addRule(new AllocationFileQueuePlacementRule("reject"))) - .writeToFile(allocFile); - + PrintWriter out = new PrintWriter(new FileWriter(allocFile)); + out.println(""); + out.println(""); + out.println(""); + out.println(" "); + out.println(" "); + out.println(""); + out.println(""); + out.println(" "); + out.println(" "); + out.println(""); + out.println(""); + out.close(); conf.set(FairSchedulerConfiguration.ALLOCATION_FILE, allocFile); } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/reservation/ReservationSystemTestUtil.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/reservation/ReservationSystemTestUtil.java index 43144ac5c19..9c138d1279b 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/reservation/ReservationSystemTestUtil.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/reservation/ReservationSystemTestUtil.java @@ -56,11 +56,6 @@ import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacitySchedulerConfiguration; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.event.NodeAddedSchedulerEvent; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.FairScheduler; - -import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair - .allocationfile.AllocationFileQueue; -import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair - .allocationfile.AllocationFileWriter; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.placement.MultiNodeSortingManager; import org.apache.hadoop.yarn.server.resourcemanager.security.ClientToAMTokenSecretManagerInRM; import org.apache.hadoop.yarn.server.resourcemanager.security.NMTokenSecretManagerInRM; @@ -109,46 +104,62 @@ public static void validateReservationQueue( .assertTrue(plan.getSharingPolicy() instanceof CapacityOverTimePolicy); } - public static void setupFSAllocationFile(String allocationFile) { - AllocationFileWriter.create() - .defaultQueueSchedulingPolicy("drf") - .addQueue(new AllocationFileQueue.Builder("default") - .weight(1).build()) - .addQueue(new AllocationFileQueue.Builder("a") - .weight(1) - .subQueue(new AllocationFileQueue.Builder("a1") - .weight(3).build()) - .subQueue(new AllocationFileQueue.Builder("a2") - .weight(7).build()) - .build()) - .addQueue(new AllocationFileQueue.Builder("dedicated") - .weight(8) - .reservation() - .build()) - .writeToFile(allocationFile); + public static void setupFSAllocationFile(String allocationFile) + throws IOException { + PrintWriter out = new PrintWriter(new FileWriter(allocationFile)); + out.println(""); + out.println(""); + out.println(""); + out.println("1"); + out.println(""); + out.println(""); + out.println("1"); + out.println(""); + out.println("3"); + out.println(""); + out.println(""); + out.println("7"); + out.println(""); + out.println(""); + out.println(""); + out.println(""); + out.println("8"); + out.println(""); + out.println( + "drf"); + out.println(""); + out.close(); } - public static void updateFSAllocationFile(String allocationFile) { - AllocationFileWriter.create() - .defaultQueueSchedulingPolicy("drf") - .addQueue(new AllocationFileQueue.Builder("default") - .weight(5).build()) - .addQueue(new AllocationFileQueue.Builder("a") - .weight(5) - .subQueue(new AllocationFileQueue.Builder("a1") - .weight(3).build()) - .subQueue(new AllocationFileQueue.Builder("a2") - .weight(7).build()) - .build()) - .addQueue(new AllocationFileQueue.Builder("dedicated") - .weight(10) - .reservation() - .build()) - .addQueue(new AllocationFileQueue.Builder("reservation") - .weight(80) - .reservation() - .build()) - .writeToFile(allocationFile); + public static void updateFSAllocationFile(String allocationFile) + throws IOException { + PrintWriter out = new PrintWriter(new FileWriter(allocationFile)); + out.println(""); + out.println(""); + out.println(""); + out.println("5"); + out.println(""); + out.println(""); + out.println("5"); + out.println(""); + out.println("3"); + out.println(""); + out.println(""); + out.println("7"); + out.println(""); + out.println(""); + out.println(""); + out.println(""); + out.println("10"); + out.println(""); + out.println(""); + out.println(""); + out.println("80"); + out.println(""); + out.println( + "drf"); + out.println(""); + out.close(); } public static FairScheduler setupFairScheduler(RMContext rmContext, diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestAllocationFileLoaderService.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestAllocationFileLoaderService.java index 129cdd416c0..a02ef551e12 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestAllocationFileLoaderService.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestAllocationFileLoaderService.java @@ -34,26 +34,24 @@ import org.apache.hadoop.yarn.server.resourcemanager.placement.SpecifiedPlacementRule; import org.apache.hadoop.yarn.server.resourcemanager.placement.UserPlacementRule; import org.apache.hadoop.yarn.server.resourcemanager.reservation.ReservationSchedulerConfiguration; - -import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.allocationfile.AllocationFileQueue; -import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.allocationfile.AllocationFileQueuePlacementPolicy; -import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.allocationfile.AllocationFileQueuePlacementRule; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.allocationfile.AllocationFileWriter; -import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.allocationfile.UserSettings; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.policies.DominantResourceFairnessPolicy; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.policies.FairSharePolicy; import org.apache.hadoop.yarn.util.ControlledClock; import org.apache.hadoop.yarn.util.SystemClock; import org.apache.hadoop.yarn.util.resource.CustomResourceTypesConfigurationProvider; import org.apache.hadoop.yarn.util.resource.Resources; -import org.junit.After; import org.junit.Before; import org.junit.Test; - import java.io.File; +import java.io.FileOutputStream; +import java.io.FileWriter; import java.io.IOException; +import java.io.OutputStreamWriter; +import java.io.PrintWriter; import java.net.URISyntaxException; import java.net.URL; +import java.nio.charset.StandardCharsets; import java.util.List; import java.util.Map; import java.util.Set; @@ -99,11 +97,6 @@ public void setup() { when(scheduler.getRMContext()).thenReturn(rmContext); } - @After - public void teardown() { - new File(ALLOC_FILE).delete(); - } - @Test public void testGetAllocationFileFromFileSystem() throws IOException, URISyntaxException { @@ -159,13 +152,18 @@ public void testGetAllocationFileFromClasspath() { @Test (timeout = 10000) public void testReload() throws Exception { - AllocationFileWriter.create() - .addQueue(new AllocationFileQueue.Builder("queueA") - .maxRunningApps(1).build()) - .addQueue(new AllocationFileQueue.Builder("queueB").build()) - .queuePlacementPolicy(new AllocationFileQueuePlacementPolicy() - .addRule(new AllocationFileQueuePlacementRule("default"))) - .writeToFile(ALLOC_FILE); + PrintWriter out = new PrintWriter(new FileWriter(ALLOC_FILE)); + out.println(""); + out.println(""); + out.println(" "); + out.println(" 1"); + out.println(" "); + out.println(" "); + out.println(" "); + out.println(" "); + out.println(" "); + out.println(""); + out.close(); ControlledClock clock = new ControlledClock(); clock.setTime(0); @@ -197,14 +195,20 @@ public void testReload() throws Exception { confHolder.allocConf = null; // Modify file and advance the clock - AllocationFileWriter.create() - .addQueue(new AllocationFileQueue.Builder("queueB") - .maxRunningApps(3).build()) - .queuePlacementPolicy(new AllocationFileQueuePlacementPolicy() - .addRule(new AllocationFileQueuePlacementRule("specified")) - .addRule(new AllocationFileQueuePlacementRule("nestedUserQueue") - .addNestedRule(new AllocationFileQueuePlacementRule("primaryGroup")))) - .writeToFile(ALLOC_FILE); + out = new PrintWriter(new FileWriter(ALLOC_FILE)); + out.println(""); + out.println(""); + out.println(" "); + out.println(" 3"); + out.println(" "); + out.println(" "); + out.println(" "); + out.println(" "); + out.println(" "); + out.println(" "); + out.println(" "); + out.println(""); + out.close(); clock.tickMsec(System.currentTimeMillis() + AllocationFileLoaderService.ALLOC_RELOAD_WAIT_MS + 10000); @@ -238,56 +242,57 @@ public void testAllocationFileParsing() throws Exception { AllocationFileLoaderService allocLoader = new AllocationFileLoaderService(scheduler); - AllocationFileWriter.create() + AllocationFileWriter + .create() // Give queue A a minimum of 1024 M - .addQueue(new AllocationFileQueue.Builder("queueA") - .minResources("1024mb,0vcores") - .maxResources("2048mb,10vcores") - .build()) + .queue("queueA") + .minResources("1024mb,0vcores") + .maxResources("2048mb,10vcores") + .buildQueue() // Give queue B a minimum of 2048 M - .addQueue(new AllocationFileQueue.Builder("queueB") + .queue("queueB") .minResources("2048mb,0vcores") .maxResources("5120mb,110vcores") .aclAdministerApps("alice,bob admins") .schedulingPolicy("fair") - .build()) + .buildQueue() // Give queue C no minimum - .addQueue(new AllocationFileQueue.Builder("queueC") - .minResources("5120mb,0vcores") - .aclSubmitApps("alice,bob admins") - .build()) + .queue("queueC") + .minResources("5120mb,0vcores") + .aclSubmitApps("alice,bob admins") + .buildQueue() // Give queue D a limit of 3 running apps and 0.4f maxAMShare - .addQueue(new AllocationFileQueue.Builder("queueD") - .maxRunningApps(3) - .maxAMShare(0.4) - .build()) + .queue("queueD") + .maxRunningApps(3) + .maxAMShare(0.4) + .buildQueue() // Give queue E a preemption timeout of one minute - .addQueue(new AllocationFileQueue.Builder("queueE") - .minSharePreemptionTimeout(60) - .build()) + .queue("queueE") + .minSharePreemptionTimeout(60) + .buildQueue() // Make queue F a parent queue without configured leaf queues // using the 'type' attribute - .addQueue(new AllocationFileQueue.Builder("queueF") - .parent(true) - .maxChildResources("2048mb,64vcores") - .build()) - .addQueue(new AllocationFileQueue.Builder("queueG") - .maxChildResources("2048mb,64vcores") - .fairSharePreemptionTimeout(120) - .minSharePreemptionTimeout(50) - .fairSharePreemptionThreshold(0.6) - .maxContainerAllocation( + .queue("queueF") + .parent(true) + .maxChildResources("2048mb,64vcores") + .buildQueue() + .queue("queueG") + .maxChildResources("2048mb,64vcores") + .fairSharePreemptionTimeout(120) + .minSharePreemptionTimeout(50) + .fairSharePreemptionThreshold(0.6) + .maxContainerAllocation( "vcores=16, memory-mb=512, " + A_CUSTOM_RESOURCE + "=10") // Create hierarchical queues G,H, with different min/fair // share preemption timeouts and preemption thresholds. // Also add a child default to make sure it doesn't impact queue H. - .subQueue(new AllocationFileQueue.Builder("queueH") - .fairSharePreemptionTimeout(180) - .minSharePreemptionTimeout(40) - .fairSharePreemptionThreshold(0.7) - .maxContainerAllocation("1024mb,8vcores") - .build()) - .build()) + .subQueue("queueH") + .fairSharePreemptionTimeout(180) + .minSharePreemptionTimeout(40) + .fairSharePreemptionThreshold(0.7) + .maxContainerAllocation("1024mb,8vcores") + .buildSubQueue() + .buildQueue() // Set default limit of apps per queue to 15 .queueMaxAppsDefault(15) // Set default limit of max resource per queue to 4G and 100 cores @@ -305,9 +310,9 @@ public void testAllocationFileParsing() throws Exception { // Set default scheduling policy to DRF .defaultQueueSchedulingPolicy("drf") // Give user1 a limit of 10 jobs - .userSettings(new UserSettings.Builder("user1") + .userSettings("user1") .maxRunningApps(10) - .build()) + .build() .writeToFile(ALLOC_FILE); allocLoader.init(conf); @@ -483,51 +488,48 @@ public void testBackwardsCompatibleAllocationFileParsing() throws Exception { AllocationFileLoaderService allocLoader = new AllocationFileLoaderService(scheduler); - AllocationFileWriter.create() - .useLegacyTagNameForQueues() - // Give queue A a minimum of 1024 M - .addQueue(new AllocationFileQueue.Builder("queueA") - .minResources("1024mb,0vcores") - .build()) - // Give queue B a minimum of 2048 M - .addQueue(new AllocationFileQueue.Builder("queueB") - .minResources("2048mb,0vcores") - .aclAdministerApps("alice,bob admins") - .build()) - // Give queue C no minimum - .addQueue(new AllocationFileQueue.Builder("queueC") - .aclAdministerApps("alice,bob admins") - .build()) - // Give queue D a limit of 3 running apps - .addQueue(new AllocationFileQueue.Builder("queueD") - .maxRunningApps(3) - .build()) - // Give queue E a preemption timeout of one minute and 0.3f threshold - .addQueue(new AllocationFileQueue.Builder("queueE") - .minSharePreemptionTimeout(60) - .fairSharePreemptionThreshold(0.3) - .build()) - // Set default limit of apps per queue to 15 - .queueMaxAppsDefault(15) - // Set default limit of apps per user to 5 - .userMaxAppsDefault(5) - // Set default limit of max resource per queue to 4G and 100 cores - .queueMaxResourcesDefault("4096mb,100vcores") - // Set default limit of AMResourceShare to 0.5f - .queueMaxAMShareDefault(0.5) - // Set default min share preemption timeout to 2 minutes - .defaultMinSharePreemptionTimeout(120) - // Set default fair share preemption timeout to 5 minutes - .defaultFairSharePreemptionTimeout(300) - // Set default fair share preemption threshold to 0.6 - .defaultFairSharePreemptionThreshold(0.6) - // Set default scheduling policy to DRF - .defaultQueueSchedulingPolicy("drf") - // Give user1 a limit of 10 jobs - .userSettings(new UserSettings.Builder("user1") - .maxRunningApps(10) - .build()) - .writeToFile(ALLOC_FILE); + PrintWriter out = new PrintWriter(new FileWriter(ALLOC_FILE)); + out.println(""); + out.println(""); + // Give queue A a minimum of 1024 M + out.println(""); + out.println("1024mb,0vcores"); + out.println(""); + // Give queue B a minimum of 2048 M + out.println(""); + out.println("2048mb,0vcores"); + out.println("alice,bob admins"); + out.println(""); + // Give queue C no minimum + out.println(""); + out.println("alice,bob admins"); + out.println(""); + // Give queue D a limit of 3 running apps + out.println(""); + out.println("3"); + out.println(""); + // Give queue E a preemption timeout of one minute and 0.3f threshold + out.println(""); + out.println("60"); + out.println("0.3"); + out.println(""); + // Set default limit of apps per queue to 15 + out.println("15"); + // Set default limit of apps per user to 5 + out.println("5"); + // Give user1 a limit of 10 jobs + out.println(""); + out.println("10"); + out.println(""); + // Set default min share preemption timeout to 2 minutes + out.println("120" + + ""); + // Set fair share preemption timeout to 5 minutes + out.println("300"); + // Set default fair share preemption threshold to 0.6f + out.println("0.6"); + out.println(""); + out.close(); allocLoader.init(conf); ReloadListener confHolder = new ReloadListener(); @@ -600,7 +602,11 @@ public void testSimplePlacementPolicyFromConf() throws Exception { conf.setBoolean(FairSchedulerConfiguration.ALLOW_UNDECLARED_POOLS, false); conf.setBoolean(FairSchedulerConfiguration.USER_AS_DEFAULT_QUEUE, false); - AllocationFileWriter.create().writeToFile(ALLOC_FILE); + PrintWriter out = new PrintWriter(new FileWriter(ALLOC_FILE)); + out.println(""); + out.println(""); + out.println(""); + out.close(); AllocationFileLoaderService allocLoader = new AllocationFileLoaderService(scheduler); @@ -626,10 +632,15 @@ public void testSimplePlacementPolicyFromConf() throws Exception { public void testQueueAlongsideRoot() throws Exception { conf.set(FairSchedulerConfiguration.ALLOCATION_FILE, ALLOC_FILE); - AllocationFileWriter.create() - .addQueue(new AllocationFileQueue.Builder("root").build()) - .addQueue(new AllocationFileQueue.Builder("other").build()) - .writeToFile(ALLOC_FILE); + PrintWriter out = new PrintWriter(new FileWriter(ALLOC_FILE)); + out.println(""); + out.println(""); + out.println(""); + out.println(""); + out.println(""); + out.println(""); + out.println(""); + out.close(); AllocationFileLoaderService allocLoader = new AllocationFileLoaderService(scheduler); @@ -647,9 +658,13 @@ public void testQueueAlongsideRoot() throws Exception { public void testQueueNameContainingPeriods() throws Exception { conf.set(FairSchedulerConfiguration.ALLOCATION_FILE, ALLOC_FILE); - AllocationFileWriter.create() - .addQueue(new AllocationFileQueue.Builder("parent1.child").build()) - .writeToFile(ALLOC_FILE); + PrintWriter out = new PrintWriter(new FileWriter(ALLOC_FILE)); + out.println(""); + out.println(""); + out.println(""); + out.println(""); + out.println(""); + out.close(); AllocationFileLoaderService allocLoader = new AllocationFileLoaderService(scheduler); @@ -667,9 +682,13 @@ public void testQueueNameContainingPeriods() throws Exception { public void testQueueNameContainingOnlyWhitespace() throws Exception { conf.set(FairSchedulerConfiguration.ALLOCATION_FILE, ALLOC_FILE); - AllocationFileWriter.create() - .addQueue(new AllocationFileQueue.Builder(" ").build()) - .writeToFile(ALLOC_FILE); + PrintWriter out = new PrintWriter(new FileWriter(ALLOC_FILE)); + out.println(""); + out.println(""); + out.println(""); + out.println(""); + out.println(""); + out.close(); AllocationFileLoaderService allocLoader = new AllocationFileLoaderService(scheduler); @@ -683,12 +702,15 @@ public void testQueueNameContainingOnlyWhitespace() throws Exception { public void testParentTagWithReservation() throws Exception { conf.set(FairSchedulerConfiguration.ALLOCATION_FILE, ALLOC_FILE); - AllocationFileWriter.create() - .addQueue(new AllocationFileQueue.Builder("parent") - .parent(true) - .reservation() - .build()) - .writeToFile(ALLOC_FILE); + PrintWriter out = new PrintWriter(new FileWriter(ALLOC_FILE)); + out.println(""); + out.println(""); + out.println(""); + out.println(""); + out.println(""); + out.println(""); + out.println(""); + out.close(); AllocationFileLoaderService allocLoader = new AllocationFileLoaderService(scheduler); @@ -709,13 +731,17 @@ public void testParentTagWithReservation() throws Exception { public void testParentWithReservation() throws Exception { conf.set(FairSchedulerConfiguration.ALLOCATION_FILE, ALLOC_FILE); - AllocationFileWriter.create() - .addQueue(new AllocationFileQueue.Builder("parent") - .parent(true) - .subQueue(new AllocationFileQueue.Builder("child").build()) - .reservation() - .build()) - .writeToFile(ALLOC_FILE); + PrintWriter out = new PrintWriter(new FileWriter(ALLOC_FILE)); + out.println(""); + out.println(""); + out.println(""); + out.println(""); + out.println(""); + out.println(" "); + out.println(" "); + out.println(""); + out.println(""); + out.close(); AllocationFileLoaderService allocLoader = new AllocationFileLoaderService(scheduler); @@ -736,12 +762,15 @@ public void testParentWithReservation() throws Exception { public void testParentTagWithChild() throws Exception { conf.set(FairSchedulerConfiguration.ALLOCATION_FILE, ALLOC_FILE); - AllocationFileWriter.create() - .addQueue(new AllocationFileQueue.Builder("parent") - .parent(true) - .subQueue(new AllocationFileQueue.Builder("child").build()) - .build()) - .writeToFile(ALLOC_FILE); + PrintWriter out = new PrintWriter(new FileWriter(ALLOC_FILE)); + out.println(""); + out.println(""); + out.println(""); + out.println(" "); + out.println(" "); + out.println(""); + out.println(""); + out.close(); AllocationFileLoaderService allocLoader = new AllocationFileLoaderService(scheduler); @@ -765,9 +794,14 @@ public void testParentTagWithChild() throws Exception { public void testQueueNameContainingNBWhitespace() throws Exception { conf.set(FairSchedulerConfiguration.ALLOCATION_FILE, ALLOC_FILE); - AllocationFileWriter.create() - .addQueue(new AllocationFileQueue.Builder("\u00a0").build()) - .writeToFile(ALLOC_FILE); + PrintWriter out = new PrintWriter(new OutputStreamWriter( + new FileOutputStream(ALLOC_FILE), StandardCharsets.UTF_8)); + out.println(""); + out.println(""); + out.println(""); + out.println(""); + out.println(""); + out.close(); AllocationFileLoaderService allocLoader = new AllocationFileLoaderService(scheduler); @@ -784,9 +818,13 @@ public void testQueueNameContainingNBWhitespace() throws Exception { public void testDefaultQueueSchedulingModeIsFIFO() throws Exception { conf.set(FairSchedulerConfiguration.ALLOCATION_FILE, ALLOC_FILE); - AllocationFileWriter.create() - .defaultQueueSchedulingPolicy("fifo") - .writeToFile(ALLOC_FILE); + PrintWriter out = new PrintWriter(new FileWriter(ALLOC_FILE)); + out.println(""); + out.println(""); + out.println("fifo" + + ""); + out.println(""); + out.close(); AllocationFileLoaderService allocLoader = new AllocationFileLoaderService(scheduler); @@ -800,14 +838,19 @@ public void testDefaultQueueSchedulingModeIsFIFO() throws Exception { public void testReservableQueue() throws Exception { conf.set(FairSchedulerConfiguration.ALLOCATION_FILE, ALLOC_FILE); - AllocationFileWriter.create() - .addQueue(new AllocationFileQueue.Builder("reservable") - .reservation() - .build()) - .addQueue(new AllocationFileQueue.Builder("other").build()) - .reservationAgent("DummyAgentName") - .reservationPolicy("AnyAdmissionPolicy") - .writeToFile(ALLOC_FILE); + PrintWriter out = new PrintWriter(new FileWriter(ALLOC_FILE)); + out.println(""); + out.println(""); + out.println(""); + out.println(""); + out.println(""); + out.println(""); + out.println(""); + out.println(""); + out.println("DummyAgentName"); + out.println("AnyAdmissionPolicy"); + out.println(""); + out.close(); AllocationFileLoaderService allocLoader = new AllocationFileLoaderService(scheduler); @@ -857,12 +900,15 @@ public void testReservableCannotBeCombinedWithDynamicUserQueue() throws Exception { conf.set(FairSchedulerConfiguration.ALLOCATION_FILE, ALLOC_FILE); - AllocationFileWriter.create() - .addQueue(new AllocationFileQueue.Builder("notboth") - .parent(true) - .reservation() - .build()) - .writeToFile(ALLOC_FILE); + PrintWriter out = new PrintWriter(new FileWriter(ALLOC_FILE)); + out.println(""); + out.println(""); + out.println(""); + out.println(""); + out.println(""); + out.println(""); + out.println(""); + out.close(); AllocationFileLoaderService allocLoader = new AllocationFileLoaderService(scheduler); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestAppRunnability.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestAppRunnability.java index 7c6411c9074..b7ef4713819 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestAppRunnability.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestAppRunnability.java @@ -43,12 +43,6 @@ import org.apache.hadoop.yarn.server.resourcemanager.scheduler.event.AppAttemptRemovedSchedulerEvent; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.event.NodeAddedSchedulerEvent; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.event.NodeUpdateSchedulerEvent; - - -import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair - .allocationfile.AllocationFileQueue; -import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair - .allocationfile.AllocationFileWriter; import org.apache.hadoop.yarn.util.resource.Resources; import org.junit.After; import org.junit.Before; @@ -170,14 +164,17 @@ public void testPreemptionVariablesForQueueCreatedRuntime() throws Exception { } @Test - public void testDontAllowUndeclaredPools() { + public void testDontAllowUndeclaredPools() throws Exception { conf.setBoolean(FairSchedulerConfiguration.ALLOW_UNDECLARED_POOLS, false); conf.set(FairSchedulerConfiguration.ALLOCATION_FILE, ALLOC_FILE); - AllocationFileWriter.create() - .addQueue(new AllocationFileQueue.Builder("jerry").build()) - .writeToFile(ALLOC_FILE); - + PrintWriter out = new PrintWriter(new FileWriter(ALLOC_FILE)); + out.println(""); + out.println(""); + out.println(""); + out.println(""); + out.println(""); + out.close(); // Restarting resource manager since the file location and content is // changed. resourceManager.stop(); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestApplicationMasterServiceWithFS.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestApplicationMasterServiceWithFS.java index a7408856941..d7ec1621910 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestApplicationMasterServiceWithFS.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestApplicationMasterServiceWithFS.java @@ -36,12 +36,6 @@ import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMApp; import org.apache.hadoop.yarn.server.resourcemanager.rmapp.attempt.RMAppAttempt; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.ResourceScheduler; - - -import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair - .allocationfile.AllocationFileQueue; -import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair - .allocationfile.AllocationFileWriter; import org.junit.AfterClass; import org.junit.Assert; import org.junit.BeforeClass; @@ -62,7 +56,7 @@ private static YarnConfiguration configuration; @BeforeClass - public static void setup() { + public static void setup() throws IOException { String allocFile = GenericTestUtils.getTestDir(TEST_FOLDER).getAbsolutePath(); @@ -71,15 +65,21 @@ public static void setup() { ResourceScheduler.class); configuration.set(FairSchedulerConfiguration.ALLOCATION_FILE, allocFile); - AllocationFileWriter.create() - .addQueue(new AllocationFileQueue.Builder("queueA") - .maxContainerAllocation("2048 mb 1 vcores") - .build()) - .addQueue(new AllocationFileQueue.Builder("queueB") - .maxContainerAllocation("3072 mb 1 vcores") - .build()) - .addQueue(new AllocationFileQueue.Builder("queueC").build()) - .writeToFile(allocFile); + PrintWriter out = new PrintWriter(new FileWriter(allocFile)); + out.println(""); + out.println(""); + out.println(" "); + out.println( + " 2048 mb 1 vcores"); + out.println(" "); + out.println(" "); + out.println( + " 3072 mb 1 vcores"); + out.println(" "); + out.println(" "); + out.println(" "); + out.println(""); + out.close(); } @AfterClass diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFSAppStarvation.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFSAppStarvation.java index 7304d68fbeb..9665f9a7bb0 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFSAppStarvation.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFSAppStarvation.java @@ -22,11 +22,6 @@ import org.apache.hadoop.yarn.server.resourcemanager.MockRM; import org.apache.hadoop.yarn.server.resourcemanager.rmnode.RMNode; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.event.NodeUpdateSchedulerEvent; - -import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair - .allocationfile.AllocationFileQueue; -import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair - .allocationfile.AllocationFileWriter; import org.apache.hadoop.yarn.util.ControlledClock; import org.junit.After; @@ -192,40 +187,52 @@ private void setupClusterAndSubmitJobs() throws Exception { * 4. Submit an app that uses up all resources on the cluster */ private void setupStarvedCluster() throws IOException { - AllocationFileWriter.create() - .defaultQueueSchedulingPolicy("drf") - // Default queue - .addQueue(new AllocationFileQueue.Builder("default").build()) - // Queue with preemption disabled - .addQueue(new AllocationFileQueue.Builder("no-preemption") - .fairSharePreemptionThreshold(0).build()) - // Queue with minshare preemption enabled - .addQueue(new AllocationFileQueue.Builder("minshare") - .fairSharePreemptionThreshold(0) - .minSharePreemptionTimeout(0) - .minResources("2048mb,2vcores") - .build()) - // FAIR queue with fairshare preemption enabled - .addQueue(new AllocationFileQueue.Builder("fairshare") - .fairSharePreemptionThreshold(1) - .fairSharePreemptionTimeout(0) - .schedulingPolicy("fair") - .subQueue(new AllocationFileQueue.Builder("child") - .fairSharePreemptionThreshold(1) - .fairSharePreemptionTimeout(0) - .schedulingPolicy("fair").build()) - .build()) - // DRF queue with fairshare preemption enabled - .addQueue(new AllocationFileQueue.Builder("drf") - .fairSharePreemptionThreshold(1) - .fairSharePreemptionTimeout(0) - .schedulingPolicy("drf") - .subQueue(new AllocationFileQueue.Builder("child") - .fairSharePreemptionThreshold(1) - .fairSharePreemptionTimeout(0) - .schedulingPolicy("drf").build()) - .build()) - .writeToFile(ALLOC_FILE.getAbsolutePath()); + PrintWriter out = new PrintWriter(new FileWriter(ALLOC_FILE)); + out.println(""); + out.println(""); + + // Default queue + out.println(""); + out.println(""); + + // Queue with preemption disabled + out.println(""); + out.println("0" + + ""); + out.println(""); + + // Queue with minshare preemption enabled + out.println(""); + out.println("0" + + ""); + out.println("0" + + ""); + out.println("2048mb,2vcores"); + out.println(""); + + // FAIR queue with fairshare preemption enabled + out.println(""); + out.println("1" + + ""); + out.println("0" + + ""); + out.println("fair"); + addChildQueue(out, "fair"); + out.println(""); + + // DRF queue with fairshare preemption enabled + out.println(""); + out.println("1" + + ""); + out.println("0" + + ""); + out.println("drf"); + addChildQueue(out, "drf"); + out.println(""); + out.println("drf" + + ""); + out.println(""); + out.close(); assertTrue("Allocation file does not exist, not running the test", ALLOC_FILE.exists()); @@ -251,6 +258,17 @@ private void setupStarvedCluster() throws IOException { assertEquals(8, scheduler.getSchedulerApp(app).getLiveContainers().size()); } + private void addChildQueue(PrintWriter out, String policy) { + // Child queue under fairshare with same settings + out.println(""); + out.println("1" + + ""); + out.println("0" + + ""); + out.println("" + policy + ""); + out.println(""); + } + private void submitAppsToEachLeafQueue() { for (String queue : QUEUES) { createSchedulingRequest(1024, 1, "root." + queue, "user", 1); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFSLeafQueue.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFSLeafQueue.java index 605cbe9989f..9c47b6be0f9 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFSLeafQueue.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFSLeafQueue.java @@ -49,8 +49,6 @@ import org.apache.hadoop.yarn.server.resourcemanager.scheduler.event.NodeAddedSchedulerEvent; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.event.NodeUpdateSchedulerEvent; import org.apache.hadoop.yarn.util.resource.ResourceUtils; -import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.allocationfile.AllocationFileQueue; -import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.allocationfile.AllocationFileWriter; import org.apache.hadoop.yarn.util.resource.Resources; import org.junit.After; import org.junit.Before; @@ -60,20 +58,6 @@ import static org.junit.Assert.assertFalse; import static org.junit.Assert.assertNotNull; -import java.io.File; -import java.io.IOException; -import java.util.ArrayList; -import java.util.Collection; -import java.util.Collections; -import java.util.List; -import java.util.concurrent.CountDownLatch; -import java.util.concurrent.ExecutorService; -import java.util.concurrent.TimeUnit; - -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertTrue; -import static org.mockito.Mockito.mock; - public class TestFSLeafQueue extends FairSchedulerTestBase { private final static String ALLOC_FILE = new File(TEST_DIR, TestFSLeafQueue.class.getName() + ".xml").getAbsolutePath(); @@ -126,14 +110,17 @@ public void testUpdateDemand() { } @Test (timeout = 5000) - public void test() { + public void test() throws Exception { conf.set(FairSchedulerConfiguration.ALLOCATION_FILE, ALLOC_FILE); - - AllocationFileWriter.create() - .queueMaxAMShareDefault(MAX_AM_SHARE) - .addQueue(new AllocationFileQueue.Builder("queueA").build()) - .addQueue(new AllocationFileQueue.Builder("queueB").build()) - .writeToFile(ALLOC_FILE); + PrintWriter out = new PrintWriter(new FileWriter(ALLOC_FILE)); + out.println(""); + out.println(""); + out.println("" + MAX_AM_SHARE + + ""); + out.println(""); + out.println(""); + out.println(""); + out.close(); resourceManager = new MockRM(conf); resourceManager.start(); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFairScheduler.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFairScheduler.java index c42d93f9a25..d9f29f1be11 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFairScheduler.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFairScheduler.java @@ -88,16 +88,6 @@ import org.apache.hadoop.yarn.server.resourcemanager.scheduler.event.NodeAddedSchedulerEvent; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.event.NodeRemovedSchedulerEvent; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.event.NodeUpdateSchedulerEvent; -import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair - .allocationfile.AllocationFileQueue; -import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair - .allocationfile.AllocationFileQueuePlacementPolicy; -import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair - .allocationfile.AllocationFileQueuePlacementRule; -import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair - .allocationfile.AllocationFileWriter; -import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair - .allocationfile.UserSettings; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.policies.DominantResourceFairnessPolicy; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.policies.FifoPolicy; import org.apache.hadoop.yarn.server.utils.BuilderUtils; @@ -348,18 +338,27 @@ public void testQueueMaximumCapacityAllocations() throws IOException { int tooHighQueueAllocation = RM_SCHEDULER_MAXIMUM_ALLOCATION_MB_VALUE +1; - AllocationFileWriter.create() - .addQueue(new AllocationFileQueue.Builder("queueA") - .maxContainerAllocation("512 mb 1 vcores").build()) - .addQueue(new AllocationFileQueue.Builder("queueB").build()) - .addQueue(new AllocationFileQueue.Builder("queueC") - .maxContainerAllocation("2048 mb 3 vcores") - .subQueue(new AllocationFileQueue.Builder("queueD").build()) - .build()) - .addQueue(new AllocationFileQueue.Builder("queueE") - .maxContainerAllocation(tooHighQueueAllocation + " mb 1 vcores") - .build()) - .writeToFile(ALLOC_FILE); + PrintWriter out = new PrintWriter(new FileWriter(ALLOC_FILE)); + out.println(""); + out.println(""); + out.println(" "); + out.println( + " 512 mb 1 vcores"); + out.println(" "); + out.println(" "); + out.println(" "); + out.println(" "); + out.println( + " 2048 mb 3 vcores"); + out.println(" "); + out.println(" "); + out.println(" "); + out.println(" "); + out.println(" " + tooHighQueueAllocation + + " mb 1 vcores"); + out.println(" "); + out.println(""); + out.close(); scheduler.init(conf); @@ -395,12 +394,17 @@ public void testNormalizationUsingQueueMaximumAllocation() int queueMaxAllocation = 4096; conf.set(FairSchedulerConfiguration.ALLOCATION_FILE, ALLOC_FILE); - AllocationFileWriter.create() - .addQueue(new AllocationFileQueue.Builder("queueA") - .maxContainerAllocation(queueMaxAllocation + " mb 1 vcores") - .build()) - .addQueue(new AllocationFileQueue.Builder("queueB").build()) - .writeToFile(ALLOC_FILE); + PrintWriter out = new PrintWriter(new FileWriter(ALLOC_FILE)); + out.println(""); + out.println(""); + out.println(" "); + out.println(" " + queueMaxAllocation + + " mb 1 vcores" + ""); + out.println(" "); + out.println(" "); + out.println(" "); + out.println(""); + out.close(); scheduler.init(conf); scheduler.start(); @@ -448,17 +452,19 @@ public void testFairShareWithHighMaxResources() throws IOException { // set queueA and queueB maxResources, // the sum of queueA and queueB maxResources is more than // Integer.MAX_VALUE. - - AllocationFileWriter.create() - .addQueue(new AllocationFileQueue.Builder("queueA") - .maxResources("1073741824 mb 1000 vcores") - .weight(.25f) - .build()) - .addQueue(new AllocationFileQueue.Builder("queueB") - .maxResources("1073741824 mb 1000 vcores") - .weight(.75f) - .build()) - .writeToFile(ALLOC_FILE); + PrintWriter out = new PrintWriter(new FileWriter(ALLOC_FILE)); + out.println(""); + out.println(""); + out.println(""); + out.println("1073741824 mb 1000 vcores"); + out.println(".25"); + out.println(""); + out.println(""); + out.println("1073741824 mb 1000 vcores"); + out.println(".75"); + out.println(""); + out.println(""); + out.close(); scheduler.init(conf); scheduler.start(); @@ -500,16 +506,18 @@ public void testFairShareWithHighMaxResources() throws IOException { public void testFairShareWithLowMaxResources() throws IOException { PrintWriter out = new PrintWriter(new FileWriter(ALLOC_FILE)); - AllocationFileWriter.create() - .addQueue(new AllocationFileQueue.Builder("queueA") - .maxResources("1024 mb 1 vcores") - .weight(.75f) - .build()) - .addQueue(new AllocationFileQueue.Builder("queueB") - .maxResources("3072 mb 3 vcores") - .weight(.25f) - .build()) - .writeToFile(ALLOC_FILE); + out.println(""); + out.println(""); + out.println(" "); + out.println(" 1024 mb 1 vcores"); + out.println(" 0.75"); + out.println(" "); + out.println(" "); + out.println(" 3072 mb 3 vcores"); + out.println(" 0.25"); + out.println(" "); + out.println(""); + out.close(); conf.set(FairSchedulerConfiguration.ALLOCATION_FILE, ALLOC_FILE); scheduler.init(conf); @@ -568,12 +576,15 @@ public void testFairShareWithLowMaxResources() throws IOException { */ @Test public void testChildMaxResources() throws IOException { - AllocationFileWriter.create() - .addQueue(new AllocationFileQueue.Builder("queueA") - .parent(true) - .maxChildResources("2048mb,2vcores") - .build()) - .writeToFile(ALLOC_FILE); + PrintWriter out = new PrintWriter(new FileWriter(ALLOC_FILE)); + + out.println(""); + out.println(""); + out.println(" "); + out.println(" 2048mb,2vcores"); + out.println(" "); + out.println(""); + out.close(); conf.set(FairSchedulerConfiguration.ALLOCATION_FILE, ALLOC_FILE); scheduler.init(conf); @@ -622,12 +633,14 @@ public void testChildMaxResources() throws IOException { assertEquals("App 2 is not running with the correct number of containers", 2, scheduler.getSchedulerApp(attId2).getLiveContainers().size()); - AllocationFileWriter.create() - .addQueue(new AllocationFileQueue.Builder("queueA") - .parent(true) - .maxChildResources("3072mb,3vcores") - .build()) - .writeToFile(ALLOC_FILE); + out = new PrintWriter(new FileWriter(ALLOC_FILE)); + out.println(""); + out.println(""); + out.println(" "); + out.println(" 3072mb,3vcores"); + out.println(" "); + out.println(""); + out.close(); scheduler.reinitialize(conf, resourceManager.getRMContext()); scheduler.update(); @@ -646,12 +659,14 @@ public void testChildMaxResources() throws IOException { assertEquals("App 2 is not running with the correct number of containers", 3, scheduler.getSchedulerApp(attId2).getLiveContainers().size()); - AllocationFileWriter.create() - .addQueue(new AllocationFileQueue.Builder("queueA") - .parent(true) - .maxChildResources("1024mb,1vcores") - .build()) - .writeToFile(ALLOC_FILE); + out = new PrintWriter(new FileWriter(ALLOC_FILE)); + out.println(""); + out.println(""); + out.println(" "); + out.println(" 1024mb,1vcores"); + out.println(" "); + out.println(""); + out.close(); //ensure that a 7th node heartbeat does not allocate more containers scheduler.handle(nodeEvent); @@ -681,13 +696,17 @@ private void drainEventsOnRM() { public void testFairShareWithZeroWeight() throws IOException { conf.set(FairSchedulerConfiguration.ALLOCATION_FILE, ALLOC_FILE); // set queueA and queueB weight zero. - - AllocationFileWriter.create() - .addQueue(new AllocationFileQueue.Builder("queueA") - .weight(0.0f).build()) - .addQueue(new AllocationFileQueue.Builder("queueB") - .weight(0.0f).build()) - .writeToFile(ALLOC_FILE); + PrintWriter out = new PrintWriter(new FileWriter(ALLOC_FILE)); + out.println(""); + out.println(""); + out.println(""); + out.println("0.0"); + out.println(""); + out.println(""); + out.println("0.0"); + out.println(""); + out.println(""); + out.close(); scheduler.init(conf); scheduler.start(); @@ -725,23 +744,26 @@ public void testFairShareWithZeroWeight() throws IOException { @Test public void testComputeMaxAMResource() throws IOException { conf.set(FairSchedulerConfiguration.ALLOCATION_FILE, ALLOC_FILE); - - AllocationFileWriter.create() - .addQueue(new AllocationFileQueue.Builder("queueFSZeroWithMax") - .weight(0) - .maxAMShare(0.5) - .maxResources("4096 mb 4 vcores") - .build()) - .addQueue(new AllocationFileQueue.Builder("queueFSZeroWithAVL") - .weight(0.0f) - .maxAMShare(0.5) - .build()) - .addQueue(new AllocationFileQueue.Builder("queueFSNonZero") - .weight(1) - .maxAMShare(0.5) - .build()) - .defaultQueueSchedulingPolicy("drf") - .writeToFile(ALLOC_FILE); + PrintWriter out = new PrintWriter(new FileWriter(ALLOC_FILE)); + out.println(""); + out.println(""); + out.println(""); + out.println("0"); + out.println("0.5"); + out.println("4096 mb 4 vcores"); + out.println(""); + out.println(""); + out.println("0.0"); + out.println("0.5"); + out.println(""); + out.println(""); + out.println("1"); + out.println("0.5"); + out.println(""); + out.println("drf" + + ""); + out.println(""); + out.close(); scheduler.init(conf); scheduler.start(); @@ -856,16 +878,19 @@ public void testFairShareWithZeroWeightNoneZeroMinRes() throws IOException { conf.set(FairSchedulerConfiguration.ALLOCATION_FILE, ALLOC_FILE); // set queueA and queueB weight zero. // set queueA and queueB minResources 1. - AllocationFileWriter.create() - .addQueue(new AllocationFileQueue.Builder("queueA") - .weight(0) - .minResources("1 mb 1 vcores") - .build()) - .addQueue(new AllocationFileQueue.Builder("queueB") - .minResources("1 mb 1 vcores") - .weight(0.0f) - .build()) - .writeToFile(ALLOC_FILE); + PrintWriter out = new PrintWriter(new FileWriter(ALLOC_FILE)); + out.println(""); + out.println(""); + out.println(""); + out.println("1 mb 1 vcores"); + out.println("0.0"); + out.println(""); + out.println(""); + out.println("1 mb 1 vcores"); + out.println("0.0"); + out.println(""); + out.println(""); + out.close(); scheduler.init(conf); scheduler.start(); @@ -903,16 +928,19 @@ public void testFairShareWithNoneZeroWeightNoneZeroMinRes() conf.set(FairSchedulerConfiguration.ALLOCATION_FILE, ALLOC_FILE); // set queueA and queueB weight 0.5. // set queueA and queueB minResources 1024. - AllocationFileWriter.create() - .addQueue(new AllocationFileQueue.Builder("queueA") - .weight(0.5f) - .minResources("1024 mb 1 vcores") - .build()) - .addQueue(new AllocationFileQueue.Builder("queueB") - .weight(0.5f) - .minResources("1024 mb 1 vcores") - .build()) - .writeToFile(ALLOC_FILE); + PrintWriter out = new PrintWriter(new FileWriter(ALLOC_FILE)); + out.println(""); + out.println(""); + out.println(""); + out.println("1024 mb 1 vcores"); + out.println("0.5"); + out.println(""); + out.println(""); + out.println("1024 mb 1 vcores"); + out.println("0.5"); + out.println(""); + out.println(""); + out.close(); scheduler.init(conf); scheduler.start(); @@ -948,14 +976,17 @@ public void testFairShareWithNoneZeroWeightNoneZeroMinRes() public void testQueueInfo() throws IOException { conf.set(FairSchedulerConfiguration.ALLOCATION_FILE, ALLOC_FILE); - AllocationFileWriter.create() - .addQueue(new AllocationFileQueue.Builder("queueA") - .weight(0.25f) - .build()) - .addQueue(new AllocationFileQueue.Builder("queueB") - .weight(0.75f) - .build()) - .writeToFile(ALLOC_FILE); + PrintWriter out = new PrintWriter(new FileWriter(ALLOC_FILE)); + out.println(""); + out.println(""); + out.println(""); + out.println(".25"); + out.println(""); + out.println(""); + out.println(".75"); + out.println(""); + out.println(""); + out.close(); scheduler.init(conf); scheduler.start(); @@ -1471,15 +1502,19 @@ public void testReservationThresholdWithAssignMultiple() throws Exception { public void testContainerReservationAttemptExceedingQueueMax() throws Exception { conf.set(FairSchedulerConfiguration.ALLOCATION_FILE, ALLOC_FILE); - - AllocationFileWriter.create() - .addQueue(new AllocationFileQueue.Builder("root") - .subQueue(new AllocationFileQueue.Builder("queue1") - .maxResources("2048mb,5vcores").build()) - .subQueue(new AllocationFileQueue.Builder("queue2") - .maxResources("2048mb,10vcores").build()) - .build()) - .writeToFile(ALLOC_FILE); + PrintWriter out = new PrintWriter(new FileWriter(ALLOC_FILE)); + out.println(""); + out.println(""); + out.println(""); + out.println(""); + out.println("2048mb,5vcores"); + out.println(""); + out.println(""); + out.println("2048mb,10vcores"); + out.println(""); + out.println(""); + out.println(""); + out.close(); scheduler.init(conf); scheduler.start(); @@ -1529,20 +1564,22 @@ public void testContainerReservationAttemptExceedingQueueMax() public void testRequestAMResourceInZeroFairShareQueue() throws Exception { conf.set(FairSchedulerConfiguration.ALLOCATION_FILE, ALLOC_FILE); - AllocationFileWriter.create() - .addQueue(new AllocationFileQueue.Builder("queue1") - .weight(0) - .maxAMShare(0.5) - .maxResources("4096mb,10vcores") - .build()) - .addQueue(new AllocationFileQueue.Builder("queue2") - .weight(2.0f) - .build()) - .addQueue(new AllocationFileQueue.Builder("queue3") - .weight(0.000001f) - .build()) - .defaultQueueSchedulingPolicy("drf") - .writeToFile(ALLOC_FILE); + PrintWriter out = new PrintWriter(new FileWriter(ALLOC_FILE)); + out.println(""); + out.println(""); + out.println(""); + out.println("0.0"); + out.println("4096mb,10vcores"); + out.println("0.5"); + out.println(""); + out.println(""); + out.println("2.0"); + out.println(""); + out.println(""); + out.println("0.000001"); + out.println(""); + out.println(""); + out.close(); scheduler.init(conf); scheduler.start(); @@ -1600,15 +1637,19 @@ public void testRequestAMResourceInZeroFairShareQueue() throws Exception { @Test (timeout = 500000) public void testContainerReservationNotExceedingQueueMax() throws Exception { conf.set(FairSchedulerConfiguration.ALLOCATION_FILE, ALLOC_FILE); - - AllocationFileWriter.create() - .addQueue(new AllocationFileQueue.Builder("root") - .subQueue(new AllocationFileQueue.Builder("queue1") - .maxResources("3072mb,10vcores").build()) - .subQueue(new AllocationFileQueue.Builder("queue2") - .maxResources("2048mb,10vcores").build()) - .build()) - .writeToFile(ALLOC_FILE); + PrintWriter out = new PrintWriter(new FileWriter(ALLOC_FILE)); + out.println(""); + out.println(""); + out.println(""); + out.println(""); + out.println("3072mb,10vcores"); + out.println(""); + out.println(""); + out.println("2048mb,10vcores"); + out.println(""); + out.println(""); + out.println(""); + out.close(); scheduler.init(conf); scheduler.start(); @@ -1657,14 +1698,19 @@ public void testContainerReservationNotExceedingQueueMax() throws Exception { getCurrentReservation().getMemorySize()); // Now reduce max Resources of queue1 down to 2048 - AllocationFileWriter.create() - .addQueue(new AllocationFileQueue.Builder("root") - .subQueue(new AllocationFileQueue.Builder("queue1") - .maxResources("2048mb,10vcores").build()) - .subQueue(new AllocationFileQueue.Builder("queue2") - .maxResources("2048mb,10vcores").build()) - .build()) - .writeToFile(ALLOC_FILE); + out = new PrintWriter(new FileWriter(ALLOC_FILE)); + out.println(""); + out.println(""); + out.println(""); + out.println(""); + out.println("2048mb,10vcores"); + out.println(""); + out.println(""); + out.println("2048mb,10vcores"); + out.println(""); + out.println(""); + out.println(""); + out.close(); scheduler.reinitialize(conf, resourceManager.getRMContext()); @@ -1685,9 +1731,13 @@ public void testContainerReservationNotExceedingQueueMax() throws Exception { public void testReservationThresholdGatesReservations() throws Exception { conf.set(FairSchedulerConfiguration.ALLOCATION_FILE, ALLOC_FILE); - AllocationFileWriter.create() - .defaultQueueSchedulingPolicy("drf") - .writeToFile(ALLOC_FILE); + PrintWriter out = new PrintWriter(new FileWriter(ALLOC_FILE)); + out.println(""); + out.println(""); + out.println("drf" + + ""); + out.println(""); + out.close(); // Set threshold to 2 * 1024 ==> 2048 MB & 2 * 1 ==> 2 vcores (test will // use vcores) @@ -1833,13 +1883,17 @@ public void testQueueuNameWithPeriods() throws Exception { public void testFairShareWithMinAlloc() throws Exception { conf.set(FairSchedulerConfiguration.ALLOCATION_FILE, ALLOC_FILE); - AllocationFileWriter.create() - .addQueue(new AllocationFileQueue.Builder("queueA") - .minResources("1024mb,0vcores").build()) - .addQueue(new AllocationFileQueue.Builder("queueB") - .minResources("2048mb,0vcores") - .build()) - .writeToFile(ALLOC_FILE); + PrintWriter out = new PrintWriter(new FileWriter(ALLOC_FILE)); + out.println(""); + out.println(""); + out.println(""); + out.println("1024mb,0vcores"); + out.println(""); + out.println(""); + out.println("2048mb,0vcores"); + out.println(""); + out.println(""); + out.close(); scheduler.init(conf); scheduler.start(); @@ -1874,18 +1928,21 @@ else if (p.getName().equals("root.queueB")) { public void testFairShareAndWeightsInNestedUserQueueRule() throws Exception { conf.set(FairSchedulerConfiguration.ALLOCATION_FILE, ALLOC_FILE); - AllocationFileWriter.create() - .addQueue(new AllocationFileQueue.Builder("parentq") - .parent(true) - .minResources("1024mb,0vcores") - .build()) - .queuePlacementPolicy(new AllocationFileQueuePlacementPolicy() - .addRule(new AllocationFileQueuePlacementRule("nestedUserQueue") - .addNestedRule( - new AllocationFileQueuePlacementRule("specified") - .create(false))) - .addRule(new AllocationFileQueuePlacementRule("default"))) - .writeToFile(ALLOC_FILE); + PrintWriter out = new PrintWriter(new FileWriter(ALLOC_FILE)); + + out.println(""); + out.println(""); + out.println(""); + out.println("1024mb,0vcores"); + out.println(""); + out.println(""); + out.println(""); + out.println(" "); + out.println(""); + out.println(""); + out.println(""); + out.println(""); + out.close(); RMApp rmApp1 = new MockRMApp(0, 0, RMAppState.NEW); RMApp rmApp2 = new MockRMApp(1, 1, RMAppState.NEW); @@ -1932,16 +1989,21 @@ public void testFairShareAndWeightsInNestedUserQueueRule() throws Exception { public void testSteadyFairShareWithReloadAndNodeAddRemove() throws Exception { conf.set(FairSchedulerConfiguration.ALLOCATION_FILE, ALLOC_FILE); - AllocationFileWriter.create() - .defaultQueueSchedulingPolicy("fair") - .addQueue(new AllocationFileQueue.Builder("root") - .schedulingPolicy("drf") - .subQueue(new AllocationFileQueue.Builder("child1") - .weight(1).build()) - .subQueue(new AllocationFileQueue.Builder("child2") - .weight(1).build()) - .build()) - .writeToFile(ALLOC_FILE); + PrintWriter out = new PrintWriter(new FileWriter(ALLOC_FILE)); + out.println(""); + out.println(""); + out.println("fair"); + out.println(""); + out.println(" drf"); + out.println(" "); + out.println(" 1"); + out.println(" "); + out.println(" "); + out.println(" 1"); + out.println(" "); + out.println(""); + out.println(""); + out.close(); scheduler.init(conf); scheduler.start(); @@ -1969,19 +2031,24 @@ public void testSteadyFairShareWithReloadAndNodeAddRemove() throws Exception { .getSteadyFairShare().getMemorySize()); // Reload the allocation configuration file - AllocationFileWriter.create() - .defaultQueueSchedulingPolicy("fair") - .addQueue(new AllocationFileQueue.Builder("root") - .schedulingPolicy("drf") - .subQueue(new AllocationFileQueue.Builder("child1") - .weight(1).build()) - .subQueue(new AllocationFileQueue.Builder("child2") - .weight(2).build()) - .subQueue(new AllocationFileQueue.Builder("child3") - .weight(2).build()) - .build()) - .writeToFile(ALLOC_FILE); - + out = new PrintWriter(new FileWriter(ALLOC_FILE)); + out.println(""); + out.println(""); + out.println("fair"); + out.println(""); + out.println(" drf"); + out.println(" "); + out.println(" 1"); + out.println(" "); + out.println(" "); + out.println(" 2"); + out.println(" "); + out.println(" "); + out.println(" 2"); + out.println(" "); + out.println(""); + out.println(""); + out.close(); scheduler.reinitialize(conf, resourceManager.getRMContext()); // The steady fair shares for all queues should be updated @@ -2101,19 +2168,27 @@ public void testQueueDemandCalculation() throws Exception { } @Test - public void testHierarchicalQueueAllocationFileParsing() throws IOException { + public void testHierarchicalQueueAllocationFileParsing() throws IOException, SAXException, + AllocationConfigurationException, ParserConfigurationException { conf.set(FairSchedulerConfiguration.ALLOCATION_FILE, ALLOC_FILE); - AllocationFileWriter.create() - .addQueue(new AllocationFileQueue.Builder("queueA") - .minResources("2048mb,0vcores").build()) - .addQueue(new AllocationFileQueue.Builder("queueB") - .minResources("2048mb,0vcores") - .subQueue(new AllocationFileQueue.Builder("queueC") - .minResources("2048mb,0vcores").build()) - .subQueue(new AllocationFileQueue.Builder("queueD") - .minResources("2048mb,0vcores").build()) - .build()) - .writeToFile(ALLOC_FILE); + + PrintWriter out = new PrintWriter(new FileWriter(ALLOC_FILE)); + out.println(""); + out.println(""); + out.println(""); + out.println("2048mb,0vcores"); + out.println(""); + out.println(""); + out.println("2048mb,0vcores"); + out.println(""); + out.println("2048mb,0vcores"); + out.println(""); + out.println(""); + out.println("2048mb,0vcores"); + out.println(""); + out.println(""); + out.println(""); + out.close(); scheduler.init(conf); scheduler.start(); @@ -2134,22 +2209,27 @@ public void testHierarchicalQueueAllocationFileParsing() throws IOException { public void testConfigureRootQueue() throws Exception { conf.set(FairSchedulerConfiguration.ALLOCATION_FILE, ALLOC_FILE); - AllocationFileWriter.create() - .defaultQueueSchedulingPolicy("fair") - .defaultFairSharePreemptionTimeout(300) - .defaultMinSharePreemptionTimeout(200) - .defaultFairSharePreemptionThreshold(.6) - .addQueue(new AllocationFileQueue.Builder("root") - .schedulingPolicy("drf") - .fairSharePreemptionTimeout(100) - .fairSharePreemptionThreshold(.5) - .minSharePreemptionTimeout(120) - .subQueue(new AllocationFileQueue.Builder("child1") - .minResources("1024mb,1vcores").build()) - .subQueue(new AllocationFileQueue.Builder("child2") - .minResources("1024mb,4vcores").build()) - .build()) - .writeToFile(ALLOC_FILE); + PrintWriter out = new PrintWriter(new FileWriter(ALLOC_FILE)); + out.println(""); + out.println(""); + out.println("fair"); + out.println(""); + out.println(" drf"); + out.println(" "); + out.println(" 1024mb,1vcores"); + out.println(" "); + out.println(" "); + out.println(" 1024mb,4vcores"); + out.println(" "); + out.println(" 100"); + out.println(" 120"); + out.println(" .5"); + out.println(""); + out.println("300"); + out.println("200"); + out.println(".6"); + out.println(""); + out.close(); scheduler.init(conf); scheduler.start(); @@ -2204,10 +2284,14 @@ public void testUserMaxRunningApps() throws Exception { // Set max running apps conf.set(FairSchedulerConfiguration.ALLOCATION_FILE, ALLOC_FILE); - AllocationFileWriter.create() - .userSettings(new UserSettings.Builder("user1") - .maxRunningApps(1).build()) - .writeToFile(ALLOC_FILE); + PrintWriter out = new PrintWriter(new FileWriter(ALLOC_FILE)); + out.println(""); + out.println(""); + out.println(""); + out.println("1"); + out.println(""); + out.println(""); + out.close(); scheduler.init(conf); scheduler.start(); @@ -2252,57 +2336,65 @@ public void testUserMaxRunningApps() throws Exception { @Test (timeout = 5000) public void testIncreaseQueueMaxRunningAppsOnTheFly() throws Exception { - AllocationFileWriter allocBefore = AllocationFileWriter.create() - .addQueue(new AllocationFileQueue.Builder("root") - .subQueue( - new AllocationFileQueue.Builder("queue1") - .maxRunningApps(1) - .build()) - .build()); - - AllocationFileWriter allocAfter = AllocationFileWriter.create() - .addQueue(new AllocationFileQueue.Builder("root") - .subQueue( - new AllocationFileQueue.Builder("queue1") - .maxRunningApps(3) - .build()) - .build()); + String allocBefore = "" + + "" + + "" + + "" + + "1" + + "" + + "" + + ""; + + String allocAfter = "" + + "" + + "" + + "" + + "3" + + "" + + "" + + ""; testIncreaseQueueSettingOnTheFlyInternal(allocBefore, allocAfter); } @Test (timeout = 5000) public void testIncreaseUserMaxRunningAppsOnTheFly() throws Exception { - AllocationFileWriter allocBefore = AllocationFileWriter.create() - .addQueue(new AllocationFileQueue.Builder("root") - .subQueue( - new AllocationFileQueue.Builder("queue1") - .maxRunningApps(10) - .build()) - .build()) - .userSettings(new UserSettings.Builder("user1") - .maxRunningApps(1).build()); - - AllocationFileWriter allocAfter = AllocationFileWriter.create() - .addQueue(new AllocationFileQueue.Builder("root") - .subQueue( - new AllocationFileQueue.Builder("queue1") - .maxRunningApps(10) - .build()) - .build()) - .userSettings(new UserSettings.Builder("user1") - .maxRunningApps(3).build()); + String allocBefore = ""+ + ""+ + ""+ + ""+ + "10"+ + ""+ + ""+ + ""+ + "1"+ + ""+ + ""; + + String allocAfter = ""+ + ""+ + ""+ + ""+ + "10"+ + ""+ + ""+ + ""+ + "3"+ + ""+ + ""; testIncreaseQueueSettingOnTheFlyInternal(allocBefore, allocAfter); } - private void testIncreaseQueueSettingOnTheFlyInternal( - AllocationFileWriter allocBefore, - AllocationFileWriter allocAfter) throws Exception { + private void testIncreaseQueueSettingOnTheFlyInternal(String allocBefore, + String allocAfter) throws Exception { // Set max running apps conf.set(FairSchedulerConfiguration.ALLOCATION_FILE, ALLOC_FILE); - allocBefore.writeToFile(ALLOC_FILE); + PrintWriter out = new PrintWriter(new FileWriter(ALLOC_FILE)); + out.println(allocBefore); + out.close(); + scheduler.init(conf); scheduler.start(); scheduler.reinitialize(conf, resourceManager.getRMContext()); @@ -2350,7 +2442,9 @@ private void testIncreaseQueueSettingOnTheFlyInternal( // App 4 should not be running assertEquals(0, scheduler.getSchedulerApp(attId4).getLiveContainers().size()); - allocAfter.writeToFile(ALLOC_FILE); + out = new PrintWriter(new FileWriter(ALLOC_FILE)); + out.println(allocAfter); + out.close(); scheduler.reinitialize(conf, resourceManager.getRMContext()); scheduler.update(); @@ -2385,56 +2479,64 @@ private void testIncreaseQueueSettingOnTheFlyInternal( @Test (timeout = 5000) public void testDecreaseQueueMaxRunningAppsOnTheFly() throws Exception { - AllocationFileWriter allocBefore = AllocationFileWriter.create() - .addQueue(new AllocationFileQueue.Builder("root") - .subQueue( - new AllocationFileQueue.Builder("queue1") - .maxRunningApps(3) - .build()) - .build()); - - AllocationFileWriter allocAfter = AllocationFileWriter.create() - .addQueue(new AllocationFileQueue.Builder("root") - .subQueue( - new AllocationFileQueue.Builder("queue1") - .maxRunningApps(1) - .build()) - .build()); + String allocBefore = "" + + "" + + "" + + "" + + "3" + + "" + + "" + + ""; + + String allocAfter = "" + + "" + + "" + + "" + + "1" + + "" + + "" + + ""; testDecreaseQueueSettingOnTheFlyInternal(allocBefore, allocAfter); } @Test (timeout = 5000) public void testDecreaseUserMaxRunningAppsOnTheFly() throws Exception { - AllocationFileWriter allocBefore = AllocationFileWriter.create() - .addQueue(new AllocationFileQueue.Builder("root") - .subQueue( - new AllocationFileQueue.Builder("queue1") - .maxRunningApps(10) - .build()) - .build()) - .userSettings(new UserSettings.Builder("user1") - .maxRunningApps(3).build()); - - AllocationFileWriter allocAfter = AllocationFileWriter.create() - .addQueue(new AllocationFileQueue.Builder("root") - .subQueue( - new AllocationFileQueue.Builder("queue1") - .maxRunningApps(10) - .build()) - .build()) - .userSettings(new UserSettings.Builder("user1") - .maxRunningApps(1).build()); + String allocBefore = ""+ + ""+ + ""+ + ""+ + "10"+ + ""+ + ""+ + ""+ + "3"+ + ""+ + ""; + + String allocAfter = ""+ + ""+ + ""+ + ""+ + "10"+ + ""+ + ""+ + ""+ + "1"+ + ""+ + ""; testDecreaseQueueSettingOnTheFlyInternal(allocBefore, allocAfter); } - private void testDecreaseQueueSettingOnTheFlyInternal( - AllocationFileWriter allocBefore, - AllocationFileWriter allocAfter) throws Exception { + private void testDecreaseQueueSettingOnTheFlyInternal(String allocBefore, + String allocAfter) throws Exception { // Set max running apps conf.set(FairSchedulerConfiguration.ALLOCATION_FILE, ALLOC_FILE); - allocBefore.writeToFile(ALLOC_FILE); + + PrintWriter out = new PrintWriter(new FileWriter(ALLOC_FILE)); + out.println(allocBefore); + out.close(); scheduler.init(conf); scheduler.start(); @@ -2483,7 +2585,9 @@ private void testDecreaseQueueSettingOnTheFlyInternal( // App 4 should not be running assertEquals(0, scheduler.getSchedulerApp(attId4).getLiveContainers().size()); - allocAfter.writeToFile(ALLOC_FILE); + out = new PrintWriter(new FileWriter(ALLOC_FILE)); + out.println(allocAfter); + out.close(); scheduler.reinitialize(conf, resourceManager.getRMContext()); scheduler.update(); @@ -2596,16 +2700,19 @@ public void testAclSubmitApplication() throws Exception { // Set acl's conf.set(FairSchedulerConfiguration.ALLOCATION_FILE, ALLOC_FILE); - AllocationFileWriter.create() - .addQueue(new AllocationFileQueue.Builder("root") - .aclSubmitApps(" ") - .aclAdministerApps(" ") - .subQueue(new AllocationFileQueue.Builder("queue1") - .aclSubmitApps("norealuserhasthisname") - .aclAdministerApps("norealuserhasthisname") - .build()) - .build()) - .writeToFile(ALLOC_FILE); + PrintWriter out = new PrintWriter(new FileWriter(ALLOC_FILE)); + out.println(""); + out.println(""); + out.println(""); + out.println(" "); + out.println(" "); + out.println(" "); + out.println(" norealuserhasthisname"); + out.println(" norealuserhasthisname"); + out.println(" "); + out.println(""); + out.println(""); + out.close(); scheduler.init(conf); scheduler.start(); @@ -2932,17 +3039,19 @@ public void testAssignContainer() throws Exception { public void testNotAllowSubmitApplication() throws Exception { // Set acl's conf.set(FairSchedulerConfiguration.ALLOCATION_FILE, ALLOC_FILE); - - AllocationFileWriter.create() - .addQueue(new AllocationFileQueue.Builder("root") - .aclSubmitApps(" ") - .aclAdministerApps(" ") - .subQueue(new AllocationFileQueue.Builder("queue1") - .aclSubmitApps("userallow") - .aclAdministerApps("userallow") - .build()) - .build()) - .writeToFile(ALLOC_FILE); + PrintWriter out = new PrintWriter(new FileWriter(ALLOC_FILE)); + out.println(""); + out.println(""); + out.println(""); + out.println(" "); + out.println(" "); + out.println(" "); + out.println(" userallow"); + out.println(" userallow"); + out.println(" "); + out.println(""); + out.println(""); + out.close(); scheduler.init(conf); scheduler.start(); @@ -3484,12 +3593,17 @@ private void verifyQueueNumRunnable(String queueName, int numRunnableInQueue, public void testUserAndQueueMaxRunningApps() throws Exception { conf.set(FairSchedulerConfiguration.ALLOCATION_FILE, ALLOC_FILE); - AllocationFileWriter.create() - .addQueue(new AllocationFileQueue.Builder("queue1") - .maxRunningApps(2).build()) - .userSettings(new UserSettings.Builder("user1") - .maxRunningApps(1).build()) - .writeToFile(ALLOC_FILE); + PrintWriter out = new PrintWriter(new FileWriter(ALLOC_FILE)); + out.println(""); + out.println(""); + out.println(""); + out.println("2"); + out.println(""); + out.println(""); + out.println("1"); + out.println(""); + out.println(""); + out.close(); scheduler.init(conf); scheduler.start(); @@ -3532,10 +3646,14 @@ public void testMultipleCompletedEvent() throws Exception { // Set up a fair scheduler conf.set(FairSchedulerConfiguration.ALLOCATION_FILE, ALLOC_FILE); - AllocationFileWriter.create() - .addQueue(new AllocationFileQueue.Builder("queue1") - .maxAMShare(0.2).build()) - .writeToFile(ALLOC_FILE); + PrintWriter out = new PrintWriter(new FileWriter(ALLOC_FILE)); + out.println(""); + out.println(""); + out.println(""); + out.println("0.2"); + out.println(""); + out.println(""); + out.close(); scheduler.init(conf); scheduler.start(); @@ -3578,10 +3696,14 @@ public void testMultipleCompletedEvent() throws Exception { public void testQueueMaxAMShare() throws Exception { conf.set(FairSchedulerConfiguration.ALLOCATION_FILE, ALLOC_FILE); - AllocationFileWriter.create() - .addQueue(new AllocationFileQueue.Builder("queue1") - .maxAMShare(0.2).build()) - .writeToFile(ALLOC_FILE); + PrintWriter out = new PrintWriter(new FileWriter(ALLOC_FILE)); + out.println(""); + out.println(""); + out.println(""); + out.println("0.2"); + out.println(""); + out.println(""); + out.close(); scheduler.init(conf); scheduler.start(); @@ -3804,18 +3926,25 @@ public void testQueueMaxAMShareDefault() throws Exception { conf.set(FairSchedulerConfiguration.ALLOCATION_FILE, ALLOC_FILE); conf.setInt(YarnConfiguration.RM_SCHEDULER_MAXIMUM_ALLOCATION_VCORES, 6); - AllocationFileWriter.create() - .defaultQueueSchedulingPolicy("fair") - .addQueue(new AllocationFileQueue.Builder("queue1").build()) - .addQueue(new AllocationFileQueue.Builder("queue2") - .maxAMShare(0.4f) - .build()) - .addQueue(new AllocationFileQueue.Builder("queue3") - .maxResources("10240 mb 4 vcores") - .build()) - .addQueue(new AllocationFileQueue.Builder("queue4").build()) - .addQueue(new AllocationFileQueue.Builder("queue5").build()) - .writeToFile(ALLOC_FILE); + PrintWriter out = new PrintWriter(new FileWriter(ALLOC_FILE)); + out.println(""); + out.println(""); + out.println(""); + out.println(""); + out.println(""); + out.println("0.4"); + out.println(""); + out.println(""); + out.println("10240 mb 4 vcores"); + out.println(""); + out.println(""); + out.println(""); + out.println(""); + out.println(""); + out.println( + "fair"); + out.println(""); + out.close(); scheduler.init(conf); scheduler.start(); @@ -3958,10 +4087,14 @@ public void testQueueMaxAMShareDefault() throws Exception { public void testQueueMaxAMShareWithContainerReservation() throws Exception { conf.set(FairSchedulerConfiguration.ALLOCATION_FILE, ALLOC_FILE); conf.setFloat(FairSchedulerConfiguration.RESERVABLE_NODES, 1f); - AllocationFileWriter.create() - .addQueue(new AllocationFileQueue.Builder("queue1") - .maxAMShare(0.5).build()) - .writeToFile(ALLOC_FILE); + PrintWriter out = new PrintWriter(new FileWriter(ALLOC_FILE)); + out.println(""); + out.println(""); + out.println(""); + out.println("0.5"); + out.println(""); + out.println(""); + out.close(); scheduler.init(conf); scheduler.start(); @@ -4220,16 +4353,19 @@ public void testMaxRunningAppsHierarchicalQueues() throws Exception { ControlledClock clock = new ControlledClock(); scheduler.setClock(clock); - AllocationFileWriter.create() - .addQueue(new AllocationFileQueue.Builder("queue1") - .maxRunningApps(3) - .subQueue(new AllocationFileQueue.Builder("sub1").build()) - .subQueue(new AllocationFileQueue.Builder("sub2").build()) - .subQueue(new AllocationFileQueue.Builder("sub3") - .maxRunningApps(1) - .build()) - .build()) - .writeToFile(ALLOC_FILE); + PrintWriter out = new PrintWriter(new FileWriter(ALLOC_FILE)); + out.println(""); + out.println(""); + out.println(""); + out.println(" 3"); + out.println(" "); + out.println(" "); + out.println(" "); + out.println(" 1"); + out.println(" "); + out.println(""); + out.println(""); + out.close(); scheduler.init(conf); scheduler.start(); @@ -4343,8 +4479,11 @@ public void testDefaultRuleInitializesProperlyWhenPolicyNotConfigured() conf.setBoolean(FairSchedulerConfiguration.ALLOW_UNDECLARED_POOLS, false); // Create an alloc file with no queue placement policy - AllocationFileWriter.create() - .writeToFile(ALLOC_FILE); + PrintWriter out = new PrintWriter(new FileWriter(ALLOC_FILE)); + out.println(""); + out.println(""); + out.println(""); + out.close(); scheduler.init(conf); scheduler.start(); @@ -4805,13 +4944,16 @@ public void testQueueNameWithTrailingSpace() throws Exception { } @Test - public void testEmptyQueueNameInConfigFile() { + public void testEmptyQueueNameInConfigFile() throws IOException { conf.set(FairSchedulerConfiguration.ALLOCATION_FILE, ALLOC_FILE); // set empty queue name - AllocationFileWriter.create() - .addQueue(new AllocationFileQueue.Builder("").build()) - .writeToFile(ALLOC_FILE); - + PrintWriter out = new PrintWriter(new FileWriter(ALLOC_FILE)); + out.println(""); + out.println(""); + out.println(""); + out.println(""); + out.println(""); + out.close(); try { scheduler.init(conf); Assert.fail("scheduler init should fail because" + @@ -5046,8 +5188,6 @@ public void testRefreshQueuesWhenRMHA() throws Exception { new HAServiceProtocol.StateChangeRequestInfo( HAServiceProtocol.RequestSource.REQUEST_BY_USER); - //ensure ALLOC_FILE contains an 'empty' config - AllocationFileWriter.create().writeToFile(ALLOC_FILE); // 1. start a standby RM, file 'ALLOC_FILE' is empty, so there is no queues MockRM rm1 = new MockRM(conf, null); rm1.init(conf); @@ -5055,10 +5195,14 @@ public void testRefreshQueuesWhenRMHA() throws Exception { rm1.getAdminService().transitionToStandby(requestInfo); // 2. add a new queue "test_queue" - AllocationFileWriter.create() - .addQueue(new AllocationFileQueue.Builder("test_queue") - .maxRunningApps(3).build()) - .writeToFile(ALLOC_FILE); + PrintWriter out = new PrintWriter(new FileWriter(ALLOC_FILE)); + out.println(""); + out.println(""); + out.println(""); + out.println(" 3"); + out.println(""); + out.println(""); + out.close(); conf.set(YarnConfiguration.RM_STORE, MemoryRMStateStore.class.getName()); // 3. start a active RM @@ -5194,12 +5338,16 @@ public void testUpdateDemand() throws IOException { public void testDumpState() throws IOException { conf.set(FairSchedulerConfiguration.ALLOCATION_FILE, ALLOC_FILE); - AllocationFileWriter.create() - .addQueue(new AllocationFileQueue.Builder("parent") - .subQueue(new AllocationFileQueue.Builder("child1") - .weight(1).build()) - .build()) - .writeToFile(ALLOC_FILE); + PrintWriter out = new PrintWriter(new FileWriter(ALLOC_FILE)); + out.println(""); + out.println(""); + out.println(""); + out.println(" "); + out.println(" 1"); + out.println(" "); + out.println(""); + out.println(""); + out.close(); ControlledClock clock = new ControlledClock(); scheduler.setClock(clock); @@ -5380,25 +5528,29 @@ private void testAppRejectedToQueueWithZeroCapacityOfResource(String resource) + "maximum queue resources: .+")); } - private void generateAllocationFileWithZeroResource(String resource) { + private void generateAllocationFileWithZeroResource(String resource) + throws IOException { + PrintWriter out = new PrintWriter(new FileWriter(ALLOC_FILE)); + out.println(""); + out.println(""); + out.println(""); + String resources = ""; if (resource.equals(ResourceInformation.MEMORY_URI)) { resources = "0 mb,2vcores"; } else if (resource.equals(ResourceInformation.VCORES_URI)) { resources = "10000 mb,0vcores"; } - - AllocationFileWriter.create() - .addQueue(new AllocationFileQueue.Builder("queueA") - .minResources(resources) - .maxResources(resources) - .weight(2.0f) - .build()) - .addQueue(new AllocationFileQueue.Builder("queueB") - .minResources("1 mb 1 vcores") - .weight(0.0f) - .build()) - .writeToFile(ALLOC_FILE); + out.println("" + resources + ""); + out.println("" + resources + ""); + out.println("2.0"); + out.println(""); + out.println(""); + out.println("1 mb 1 vcores"); + out.println("0.0"); + out.println(""); + out.println(""); + out.close(); } @Test diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFairSchedulerFairShare.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFairSchedulerFairShare.java index c1925fda6f6..a79aacc196e 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFairSchedulerFairShare.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFairSchedulerFairShare.java @@ -33,11 +33,6 @@ import org.apache.hadoop.yarn.server.resourcemanager.rmnode.RMNode; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.event.AppAttemptRemovedSchedulerEvent; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.event.NodeAddedSchedulerEvent; - -import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair - .allocationfile.AllocationFileQueue; -import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair - .allocationfile.AllocationFileWriter; import org.apache.hadoop.yarn.util.resource.Resources; import org.junit.After; import org.junit.Before; @@ -62,29 +57,34 @@ public void teardown() { conf = null; } - private void createClusterWithQueuesAndOneNode(int mem, String policy) { + private void createClusterWithQueuesAndOneNode(int mem, String policy) + throws IOException { createClusterWithQueuesAndOneNode(mem, 0, policy); } private void createClusterWithQueuesAndOneNode(int mem, int vCores, - String policy) { - AllocationFileWriter.create() - .defaultQueueSchedulingPolicy(policy) - .addQueue(new AllocationFileQueue.Builder("root") - .subQueue(new AllocationFileQueue.Builder("parentA") - .weight(8) - .subQueue(new AllocationFileQueue.Builder("childA1").build()) - .subQueue(new AllocationFileQueue.Builder("childA2").build()) - .subQueue(new AllocationFileQueue.Builder("childA3").build()) - .subQueue(new AllocationFileQueue.Builder("childA4").build()) - .build()) - .subQueue(new AllocationFileQueue.Builder("parentB") - .weight(1) - .subQueue(new AllocationFileQueue.Builder("childB1").build()) - .subQueue(new AllocationFileQueue.Builder("childB2").build()) - .build()) - .build()) - .writeToFile(ALLOC_FILE); + String policy) throws IOException { + PrintWriter out = new PrintWriter(new FileWriter(ALLOC_FILE)); + out.println(""); + out.println(""); + out.println(""); + out.println(" "); + out.println(" 8"); + out.println(" "); + out.println(" "); + out.println(" "); + out.println(" "); + out.println(" "); + out.println(" "); + out.println(" 1"); + out.println(" "); + out.println(" "); + out.println(" "); + out.println(""); + out.println("" + policy + + ""); + out.println(""); + out.close(); resourceManager = new MockRM(conf); resourceManager.start(); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFairSchedulerQueueACLs.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFairSchedulerQueueACLs.java index 041b09e3667..32aab5531f8 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFairSchedulerQueueACLs.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFairSchedulerQueueACLs.java @@ -25,34 +25,33 @@ import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.yarn.conf.YarnConfiguration; import org.apache.hadoop.yarn.server.resourcemanager.QueueACLsTestBase; -import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair - .allocationfile.AllocationFileQueue; -import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair - .allocationfile.AllocationFileWriter; public class TestFairSchedulerQueueACLs extends QueueACLsTestBase { @Override - protected Configuration createConfiguration() { + protected Configuration createConfiguration() throws IOException { FairSchedulerConfiguration fsConf = new FairSchedulerConfiguration(); final String TEST_DIR = new File(System.getProperty("test.build.data", "/tmp")).getAbsolutePath(); final String ALLOC_FILE = new File(TEST_DIR, "test-queues.xml") .getAbsolutePath(); - - AllocationFileWriter.create() - .addQueue(new AllocationFileQueue.Builder("root") - .aclSubmitApps(" ") - .aclAdministerApps("root_admin ") - .subQueue(new AllocationFileQueue.Builder("queueA") - .aclSubmitApps("queueA_user,common_user ") - .aclAdministerApps("queueA_admin ").build()) - .subQueue(new AllocationFileQueue.Builder("queueB") - .aclSubmitApps("queueB_user,common_user ") - .aclAdministerApps("queueB_admin ").build()) - .build()) - .writeToFile(ALLOC_FILE); - + PrintWriter out = new PrintWriter(new FileWriter(ALLOC_FILE)); + out.println(""); + out.println(""); + out.println(""); + out.println(" "); + out.println(" root_admin "); + out.println(" "); + out.println(" queueA_user,common_user "); + out.println(" queueA_admin "); + out.println(" "); + out.println(" "); + out.println(" queueB_user,common_user "); + out.println(" queueB_admin "); + out.println(" "); + out.println(""); + out.println(""); + out.close(); fsConf.set(FairSchedulerConfiguration.ALLOCATION_FILE, ALLOC_FILE); fsConf.setBoolean(YarnConfiguration.YARN_ACL_ENABLE, true); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestQueueManagerRealScheduler.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestQueueManagerRealScheduler.java index 4e4cc7a3d8c..5736f75bbe5 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestQueueManagerRealScheduler.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestQueueManagerRealScheduler.java @@ -18,10 +18,6 @@ package org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair; import org.apache.hadoop.yarn.server.resourcemanager.MockRM; -import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair - .allocationfile.AllocationFileQueue; -import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair - .allocationfile.AllocationFileWriter; import org.junit.After; import org.junit.Before; import org.junit.Test; @@ -42,7 +38,7 @@ @Before public void setup() throws IOException { createConfiguration(); - writeAllocFile(30); + writeAllocFile(30, 40); conf.set(FairSchedulerConfiguration.ALLOCATION_FILE, ALLOC_FILE.getAbsolutePath()); @@ -60,20 +56,32 @@ public void teardown() { } } - private void writeAllocFile(int defaultFairShareTimeout) { - AllocationFileWriter.create() - .addQueue(new AllocationFileQueue.Builder("default") - .build()) - .addQueue(new AllocationFileQueue.Builder("queueA").build()) - .addQueue(new AllocationFileQueue.Builder("queueB") - .subQueue(new AllocationFileQueue.Builder("queueB1") - .minSharePreemptionTimeout(5).build()) - .subQueue(new AllocationFileQueue.Builder("queueB2").build()) - .build()) - .addQueue(new AllocationFileQueue.Builder("queueC").build()) - .defaultMinSharePreemptionTimeout(15) - .defaultFairSharePreemptionTimeout(defaultFairShareTimeout) - .writeToFile(ALLOC_FILE.getAbsolutePath()); + private void writeAllocFile(int defaultFairShareTimeout, + int fairShareTimeout) throws IOException { + PrintWriter out = new PrintWriter(new FileWriter(ALLOC_FILE)); + out.println(""); + out.println(""); + out.println(""); + out.println(""); + out.println(""); + out.println(""); + out.println(""); + out.println(""); + out.println("5"); + out.println(""); + out.println(""); + out.println(""); + out.println(""); + out.println(""); + out.println(""); + out.println("15" + + ""); + out.println("" + + + defaultFairShareTimeout + ""); + out.println("" + + fairShareTimeout + ""); + out.println(""); + out.close(); } @Test @@ -112,7 +120,7 @@ public void testBackwardsCompatiblePreemptionConfiguration() // Lower the fairshare preemption timeouts and verify it is picked // correctly. - writeAllocFile(25); + writeAllocFile(25, 30); scheduler.reinitialize(conf, resourceManager.getRMContext()); assertEquals(25000, queueMgr.getQueue("root") .getFairSharePreemptionTimeout()); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestSchedulingPolicy.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestSchedulingPolicy.java index cc0ab5630c1..e2be0d95adc 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestSchedulingPolicy.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestSchedulingPolicy.java @@ -32,10 +32,6 @@ import org.apache.hadoop.yarn.api.records.Resource; import org.apache.hadoop.yarn.server.resourcemanager.RMContext; import org.apache.hadoop.yarn.server.resourcemanager.placement.PlacementManager; -import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair - .allocationfile.AllocationFileQueue; -import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair - .allocationfile.AllocationFileWriter; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.policies.DominantResourceFairnessPolicy; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.policies.FairSharePolicy; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.policies.FifoPolicy; @@ -323,16 +319,22 @@ public boolean isPreemptable() { @Test public void testSchedulingPolicyViolation() throws IOException { conf.set(FairSchedulerConfiguration.ALLOCATION_FILE, ALLOC_FILE); - AllocationFileWriter.create() - .defaultQueueSchedulingPolicy("drf") - .addQueue(new AllocationFileQueue.Builder("root") - .schedulingPolicy("fair") - .subQueue(new AllocationFileQueue.Builder("child1") - .schedulingPolicy("drf").build()) - .subQueue(new AllocationFileQueue.Builder("child2") - .schedulingPolicy("fair").build()) - .build()) - .writeToFile(ALLOC_FILE); + PrintWriter out = new PrintWriter(new FileWriter(ALLOC_FILE)); + out.println(""); + out.println(""); + out.println(""); + out.println("fair"); + out.println(" "); + out.println(" drf"); + out.println(" "); + out.println(" "); + out.println(" fair"); + out.println(" "); + out.println(""); + out.println("drf" + + ""); + out.println(""); + out.close(); scheduler.init(conf); @@ -347,16 +349,22 @@ public void testSchedulingPolicyViolation() throws IOException { + " policy if its parent policy is 'fair'.", dynamicQueue); // Set child1 to 'fair' and child2 to 'drf', the reload the allocation file. - AllocationFileWriter.create() - .defaultQueueSchedulingPolicy("drf") - .addQueue(new AllocationFileQueue.Builder("root") - .schedulingPolicy("fair") - .subQueue(new AllocationFileQueue.Builder("child1") - .schedulingPolicy("fair").build()) - .subQueue(new AllocationFileQueue.Builder("child2") - .schedulingPolicy("drf").build()) - .build()) - .writeToFile(ALLOC_FILE); + out = new PrintWriter(new FileWriter(ALLOC_FILE)); + out.println(""); + out.println(""); + out.println(""); + out.println("fair"); + out.println(" "); + out.println(" fair"); + out.println(" "); + out.println(" "); + out.println(" drf"); + out.println(" "); + out.println(""); + out.println("drf" + + ""); + out.println(""); + out.close(); scheduler.reinitialize(conf, null); child1 = scheduler.getQueueManager().getQueue("child1"); @@ -371,21 +379,26 @@ public void testSchedulingPolicyViolation() throws IOException { } @Test - public void testSchedulingPolicyViolationInTheMiddleLevel() { + public void testSchedulingPolicyViolationInTheMiddleLevel() + throws IOException { conf.set(FairSchedulerConfiguration.ALLOCATION_FILE, ALLOC_FILE); - AllocationFileWriter.create() - .addQueue(new AllocationFileQueue.Builder("root") - .schedulingPolicy("fair") - .subQueue(new AllocationFileQueue.Builder("level2") - .schedulingPolicy("fair") - .subQueue(new AllocationFileQueue.Builder("level3") - .schedulingPolicy("drf") - .subQueue(new AllocationFileQueue.Builder("leaf") - .schedulingPolicy("fair").build()) - .build()) - .build()) - .build()) - .writeToFile(ALLOC_FILE); + PrintWriter out = new PrintWriter(new FileWriter(ALLOC_FILE)); + out.println(""); + out.println(""); + out.println(""); + out.println("fair"); + out.println(" "); + out.println(" fair"); + out.println(" "); + out.println(" drf"); + out.println(" "); + out.println(" fair"); + out.println(" "); + out.println(" "); + out.println(" "); + out.println(""); + out.println(""); + out.close(); scheduler.init(conf); @@ -404,16 +417,19 @@ public void testSchedulingPolicyViolationInTheMiddleLevel() { public void testFIFOPolicyOnlyForLeafQueues() throws IOException { conf.set(FairSchedulerConfiguration.ALLOCATION_FILE, ALLOC_FILE); - - AllocationFileWriter.create() - .addQueue(new AllocationFileQueue.Builder("root") - .subQueue(new AllocationFileQueue.Builder("intermediate") - .schedulingPolicy("fifo") - .subQueue(new AllocationFileQueue.Builder("leaf") - .schedulingPolicy("fair").build()) - .build()) - .build()) - .writeToFile(ALLOC_FILE); + PrintWriter out = new PrintWriter(new FileWriter(ALLOC_FILE)); + out.println(""); + out.println(""); + out.println(""); + out.println(" "); + out.println(" fifo"); + out.println(" "); + out.println(" fair"); + out.println(" "); + out.println(" "); + out.println(""); + out.println(""); + out.close(); scheduler.init(conf); @@ -421,15 +437,19 @@ public void testFIFOPolicyOnlyForLeafQueues() assertNull("Queue 'intermediate' should be null since 'fifo' is only for " + "leaf queue.", intermediate); - AllocationFileWriter.create() - .addQueue(new AllocationFileQueue.Builder("root") - .subQueue(new AllocationFileQueue.Builder("intermediate") - .schedulingPolicy("fair") - .subQueue(new AllocationFileQueue.Builder("leaf") - .schedulingPolicy("fifo").build()) - .build()) - .build()) - .writeToFile(ALLOC_FILE); + out = new PrintWriter(new FileWriter(ALLOC_FILE)); + out.println(""); + out.println(""); + out.println(""); + out.println(" "); + out.println(" fair"); + out.println(" "); + out.println(" fifo"); + out.println(" "); + out.println(" "); + out.println(""); + out.println(""); + out.close(); scheduler.reinitialize(conf, null); @@ -443,28 +463,39 @@ public void testFIFOPolicyOnlyForLeafQueues() @Test public void testPolicyReinitilization() throws IOException { conf.set(FairSchedulerConfiguration.ALLOCATION_FILE, ALLOC_FILE); - AllocationFileWriter.create() - .addQueue(new AllocationFileQueue.Builder("root") - .schedulingPolicy("fair") - .subQueue(new AllocationFileQueue.Builder("child1") - .schedulingPolicy("fair").build()) - .subQueue(new AllocationFileQueue.Builder("child2") - .schedulingPolicy("fair").build()) - .build()) - .writeToFile(ALLOC_FILE); + PrintWriter out = new PrintWriter(new FileWriter(ALLOC_FILE)); + out.println(""); + out.println(""); + out.println(""); + out.println("fair"); + out.println(" "); + out.println(" fair"); + out.println(" "); + out.println(" "); + out.println(" fair"); + out.println(" "); + out.println(""); + out.println(""); + out.close(); scheduler.init(conf); // Set child1 to 'drf' which is not allowed, then reload the allocation file - AllocationFileWriter.create() - .addQueue(new AllocationFileQueue.Builder("root") - .schedulingPolicy("fair") - .subQueue(new AllocationFileQueue.Builder("child1") - .schedulingPolicy("drf").build()) - .subQueue(new AllocationFileQueue.Builder("child2") - .schedulingPolicy("fifo").build()) - .build()) - .writeToFile(ALLOC_FILE); + out = new PrintWriter(new FileWriter(ALLOC_FILE)); + out.println(""); + out.println(""); + out.println(""); + out.println("fair"); + out.println(" "); + out.println(" drf"); + out.println(" "); + out.println(" "); + out.println(" fifo"); + out.println(" "); + out.println(""); + out.println(""); + out.close(); + scheduler.reinitialize(conf, null); FSQueue child1 = scheduler.getQueueManager().getQueue("child1"); @@ -477,15 +508,20 @@ public void testPolicyReinitilization() throws IOException { child2.getPolicy() instanceof FairSharePolicy); // Set both child1 and root to 'drf', then reload the allocation file - AllocationFileWriter.create() - .addQueue(new AllocationFileQueue.Builder("root") - .schedulingPolicy("drf") - .subQueue(new AllocationFileQueue.Builder("child1") - .schedulingPolicy("drf").build()) - .subQueue(new AllocationFileQueue.Builder("child2") - .schedulingPolicy("fifo").build()) - .build()) - .writeToFile(ALLOC_FILE); + out = new PrintWriter(new FileWriter(ALLOC_FILE)); + out.println(""); + out.println(""); + out.println(""); + out.println("drf"); + out.println(" "); + out.println(" drf"); + out.println(" "); + out.println(" "); + out.println(" fifo"); + out.println(" "); + out.println(""); + out.println(""); + out.close(); scheduler.reinitialize(conf, null); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/allocationfile/AllocationFileQueue.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/allocationfile/AllocationFileQueue.java index 0dfd6511f2f..db816130e0e 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/allocationfile/AllocationFileQueue.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/allocationfile/AllocationFileQueue.java @@ -16,249 +16,69 @@ package org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.allocationfile; -import com.google.common.collect.Lists; - import java.io.PrintWriter; import java.io.StringWriter; import java.util.List; -import static org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair - .allocationfile.AllocationFileWriter.addIfPresent; - -public class AllocationFileQueue { - private static final String DEFAULT_TAG_NAME = "queue"; - private static final String LEGACY_TAG_NAME = "pool"; - - private final String queueName; - private final String minResources; - private final String maxResources; - private final String aclAdministerApps; - private final String aclSubmitApps; - private final String aclSubmitReservations; - private final String aclAdministerReservations; - private final String aclListReservations; - private final String schedulingPolicy; - private final Integer maxRunningApps; - private final Double maxAMShare; - private final Integer minSharePreemptionTimeout; - private final String maxChildResources; - private final Integer fairSharePreemptionTimeout; - private final Double fairSharePreemptionThreshold; - private final String maxContainerAllocation; +class AllocationFileQueue { + private final AllocationFileQueueProperties properties; private final List subQueues; - private final Float weight; - private String tagName; - - private final boolean parent; - private final boolean reservation; - private AllocationFileQueue(Builder builder) { - this.queueName = builder.name; - this.parent = builder.parent; - this.minResources = builder.minResources; - this.maxResources = builder.maxResources; - this.aclAdministerApps = builder.aclAdministerApps; - this.aclSubmitApps = builder.aclSubmitApps; - this.aclSubmitReservations = builder.aclSubmitReservations; - this.aclAdministerReservations = builder.aclAdministerReservations; - this.aclListReservations = builder.aclListReservations; - this.schedulingPolicy = builder.schedulingPolicy; - this.maxRunningApps = builder.maxRunningApps; - this.maxAMShare = builder.maxAMShare; - this.minSharePreemptionTimeout = builder.minSharePreemptionTimeout; - this.maxChildResources = builder.maxChildResources; - this.fairSharePreemptionTimeout = builder.fairSharePreemptionTimeout; - this.fairSharePreemptionThreshold = builder.fairSharePreemptionThreshold; - this.maxContainerAllocation = builder.maxContainerAllocation; - this.weight = builder.weight; - this.reservation = builder.reservation; - this.subQueues = builder.subQueues; - this.tagName = DEFAULT_TAG_NAME; + AllocationFileQueue(AllocationFileQueueProperties properties, + List subQueues) { + this.properties = properties; + this.subQueues = subQueues; } String render() { StringWriter sw = new StringWriter(); PrintWriter pw = new PrintWriter(sw); printStartTag(pw); - AllocationFileWriter.printQueues(pw, subQueues, - tagName.equals(LEGACY_TAG_NAME)); - addIfPresent(pw, "minResources", minResources); - addIfPresent(pw, "maxResources", maxResources); - addIfPresent(pw, "aclAdministerApps", aclAdministerApps); - addIfPresent(pw, "aclSubmitApps", aclSubmitApps); - addIfPresent(pw, "aclSubmitReservations", aclSubmitReservations); - addIfPresent(pw, "aclAdministerReservations", aclAdministerReservations); - addIfPresent(pw, "aclListReservations", aclListReservations); - addIfPresent(pw, "schedulingPolicy", schedulingPolicy); - addIfPresent(pw, "maxRunningApps", maxRunningApps); - addIfPresent(pw, "maxAMShare", maxAMShare); - addIfPresent(pw, "minSharePreemptionTimeout", minSharePreemptionTimeout); - addIfPresent(pw, "maxChildResources", maxChildResources); - addIfPresent(pw, "fairSharePreemptionTimeout", fairSharePreemptionTimeout); - addIfPresent(pw, "fairSharePreemptionThreshold", - fairSharePreemptionThreshold); - addIfPresent(pw, "maxContainerAllocation", maxContainerAllocation); - addIfPresent(pw, "weight", weight); - if (reservation) { - pw.println(""); - } + AllocationFileWriter.printQueues(pw, subQueues); + AllocationFileWriter.addIfPresent(pw, "minResources", + properties::getMinResources); + AllocationFileWriter.addIfPresent(pw, "maxResources", + properties::getMaxResources); + AllocationFileWriter.addIfPresent(pw, "aclAdministerApps", + properties::getAclAdministerApps); + AllocationFileWriter.addIfPresent(pw, "aclSubmitApps", + properties::getAclSubmitApps); + AllocationFileWriter.addIfPresent(pw, "schedulingPolicy", + properties::getSchedulingPolicy); + AllocationFileWriter.addIfPresent(pw, "maxRunningApps", + () -> AllocationFileWriter + .createNumberSupplier(properties.getMaxRunningApps())); + AllocationFileWriter.addIfPresent(pw, "maxAMShare", + () -> AllocationFileWriter.createNumberSupplier(properties + .getMaxAMShare())); + AllocationFileWriter.addIfPresent(pw, "minSharePreemptionTimeout", + () -> AllocationFileWriter + .createNumberSupplier(properties.getMinSharePreemptionTimeout())); + AllocationFileWriter.addIfPresent(pw, "maxChildResources", + properties::getMaxChildResources); + AllocationFileWriter.addIfPresent(pw, "fairSharePreemptionTimeout", + () -> AllocationFileWriter + .createNumberSupplier(properties.getFairSharePreemptionTimeout())); + AllocationFileWriter.addIfPresent(pw, "fairSharePreemptionThreshold", + () -> AllocationFileWriter.createNumberSupplier( + properties.getFairSharePreemptionThreshold())); + AllocationFileWriter.addIfPresent(pw, "maxContainerAllocation", + () -> AllocationFileWriter + .createNumberSupplier(properties.getMaxContainerAllocation())); printEndTag(pw); pw.close(); - return sw.toString(); } - String renderWithLegacyTag() { - this.tagName = LEGACY_TAG_NAME; - return render(); - } - private void printStartTag(PrintWriter pw) { - String queueWithName = String.format("<%s name=\"%s\"", tagName, queueName); - pw.print(queueWithName); - if (parent) { - pw.print(" type=\"parent\""); + pw.print(""); } private void printEndTag(PrintWriter pw) { - pw.println(""); - } - - /** - * Class that can build queues (with subqueues) for testcases. - * The intention of having this class to group the common properties of - * simple queues and subqueues by methods delegating calls to a - * queuePropertiesBuilder instance. - */ - public static class Builder { - String name; - String minResources; - String maxResources; - String aclAdministerApps; - String aclSubmitApps; - String aclSubmitReservations; - String aclAdministerReservations; - String aclListReservations; - String schedulingPolicy; - Integer maxRunningApps; - Double maxAMShare; - Integer minSharePreemptionTimeout; - boolean parent; - String maxChildResources; - Integer fairSharePreemptionTimeout; - Double fairSharePreemptionThreshold; - String maxContainerAllocation; - boolean reservation; - final List subQueues = Lists.newArrayList(); - private Float weight; - - public Builder(String name) { - this.name = name; - } - - public Builder parent(boolean parent) { - this.parent = parent; - return this; - } - - public Builder minResources(String value) { - this.minResources = value; - return this; - } - - public Builder maxResources(String value) { - this.maxResources = value; - return this; - } - - public Builder aclAdministerApps(String value) { - this.aclAdministerApps = value; - return this; - } - - public Builder aclSubmitApps(String value) { - this.aclSubmitApps = value; - return this; - } - - public Builder aclSubmitReservations(String value) { - this.aclSubmitReservations = value; - return this; - } - - public Builder aclAdministerReservations(String value) { - this.aclAdministerReservations = value; - return this; - } - - public Builder aclListReservations(String value) { - this.aclListReservations = value; - return this; - } - - public Builder schedulingPolicy(String value) { - this.schedulingPolicy = value; - return this; - } - - public Builder maxRunningApps(int value) { - this.maxRunningApps = value; - return this; - } - - public Builder maxAMShare(double value) { - this.maxAMShare = value; - return this; - } - - public Builder minSharePreemptionTimeout(int value) { - this.minSharePreemptionTimeout = value; - return this; - } - - public Builder maxChildResources(String value) { - this.maxChildResources = value; - return this; - } - - public Builder fairSharePreemptionTimeout(Integer value) { - this.fairSharePreemptionTimeout = value; - return this; - } - - public Builder fairSharePreemptionThreshold( - double value) { - this.fairSharePreemptionThreshold = value; - return this; - } - - public Builder maxContainerAllocation(String value) { - this.maxContainerAllocation = value; - return this; - } - - public Builder weight(float value) { - this.weight = value; - return this; - } - - public Builder reservation() { - this.reservation = true; - return this; - } - - public Builder subQueue(AllocationFileQueue queue) { - if (queue == null) { - throw new IllegalArgumentException("Subqueue cannot be null!"); - } - subQueues.add(queue); - return this; - } - - public AllocationFileQueue build() { - return new AllocationFileQueue(this); - } + pw.println(""); } } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/allocationfile/AllocationFileQueueBuilder.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/allocationfile/AllocationFileQueueBuilder.java new file mode 100644 index 00000000000..176024e9b46 --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/allocationfile/AllocationFileQueueBuilder.java @@ -0,0 +1,121 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * http://www.apache.org/licenses/LICENSE-2.0 + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.allocationfile; + +/** + * Abstract base class for building simple queues and subqueues for testcases. + * Currently there are two concrete types subclassed from this class: + * {@link AllocationFileSimpleQueueBuilder} and + * {@link AllocationFileSubQueueBuilder}. + * The intention of having this class to group the common properties of + * simple queues and subqueues by methods delegating calls to a + * queuePropertiesBuilder instance. + */ +public abstract class AllocationFileQueueBuilder { + final AllocationFileQueueProperties.Builder queuePropertiesBuilder; + + AllocationFileQueueBuilder() { + this.queuePropertiesBuilder = + AllocationFileQueueProperties.Builder.create(); + } + + public AllocationFileQueueBuilder parent(boolean parent) { + this.queuePropertiesBuilder.parent(parent); + return this; + } + + public AllocationFileQueueBuilder minResources(String value) { + this.queuePropertiesBuilder.minResources(value); + return this; + } + + public AllocationFileQueueBuilder maxResources(String value) { + this.queuePropertiesBuilder.maxResources(value); + return this; + } + + public AllocationFileQueueBuilder aclAdministerApps(String value) { + this.queuePropertiesBuilder.aclAdministerApps(value); + return this; + } + + public AllocationFileQueueBuilder aclSubmitApps(String value) { + this.queuePropertiesBuilder.aclSubmitApps(value); + return this; + } + + public AllocationFileQueueBuilder schedulingPolicy(String value) { + this.queuePropertiesBuilder.schedulingPolicy(value); + return this; + } + + public AllocationFileQueueBuilder maxRunningApps(int value) { + this.queuePropertiesBuilder.maxRunningApps(value); + return this; + } + + public AllocationFileQueueBuilder maxAMShare(double value) { + this.queuePropertiesBuilder.maxAMShare(value); + return this; + } + + public AllocationFileQueueBuilder minSharePreemptionTimeout(int value) { + this.queuePropertiesBuilder.minSharePreemptionTimeout(value); + return this; + } + + public AllocationFileQueueBuilder maxChildResources(String value) { + this.queuePropertiesBuilder.maxChildResources(value); + return this; + } + + public AllocationFileQueueBuilder fairSharePreemptionTimeout(Integer value) { + this.queuePropertiesBuilder.fairSharePreemptionTimeout(value); + return this; + } + + public AllocationFileQueueBuilder fairSharePreemptionThreshold( + double value) { + this.queuePropertiesBuilder.fairSharePreemptionThreshold(value); + return this; + } + + public AllocationFileQueueBuilder maxContainerAllocation( + String maxContainerAllocation) { + this.queuePropertiesBuilder.maxContainerAllocation(maxContainerAllocation); + return this; + } + + public AllocationFileQueueBuilder subQueue(String queueName) { + if (this instanceof AllocationFileSimpleQueueBuilder) { + return new AllocationFileSubQueueBuilder( + (AllocationFileSimpleQueueBuilder) this, queueName); + } else { + throw new IllegalStateException( + "subQueue can only be invoked on instances of " + + AllocationFileSimpleQueueBuilder.class); + } + } + + public abstract AllocationFileWriter buildQueue(); + + public abstract AllocationFileSimpleQueueBuilder buildSubQueue(); + + AllocationFileQueueProperties.Builder getqueuePropertiesBuilder() { + return queuePropertiesBuilder; + } +} diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/allocationfile/AllocationFileQueuePlacementPolicy.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/allocationfile/AllocationFileQueuePlacementPolicy.java deleted file mode 100644 index 0d0b9565bff..00000000000 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/allocationfile/AllocationFileQueuePlacementPolicy.java +++ /dev/null @@ -1,59 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * http://www.apache.org/licenses/LICENSE-2.0 - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.allocationfile; - - -import com.google.common.collect.Lists; - -import java.io.PrintWriter; -import java.io.StringWriter; -import java.util.List; - -public class AllocationFileQueuePlacementPolicy { - private List rules = Lists.newArrayList(); - - public AllocationFileQueuePlacementPolicy addRule( - AllocationFileQueuePlacementRule rule) { - this.rules.add(rule); - return this; - } - - public String render() { - StringWriter sw = new StringWriter(); - PrintWriter pw = new PrintWriter(sw); - addStartTag(pw); - addRules(pw); - addEndTag(pw); - pw.close(); - - return sw.toString(); - } - - private void addStartTag(PrintWriter pw) { - pw.println(""); - } - - private void addRules(PrintWriter pw) { - for (AllocationFileQueuePlacementRule rule : rules) { - pw.println(rule.render()); - } - } - - private void addEndTag(PrintWriter pw) { - pw.println(""); - } -} diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/allocationfile/AllocationFileQueuePlacementRule.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/allocationfile/AllocationFileQueuePlacementRule.java deleted file mode 100644 index cbc1d872534..00000000000 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/allocationfile/AllocationFileQueuePlacementRule.java +++ /dev/null @@ -1,84 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * http://www.apache.org/licenses/LICENSE-2.0 - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.allocationfile; - - -import com.google.common.collect.Lists; - -import java.io.PrintWriter; -import java.io.StringWriter; -import java.util.List; - -public class AllocationFileQueuePlacementRule { - private String name; - private boolean create = true; - private String queue; - private List nestedRules = Lists - .newArrayList(); - - public AllocationFileQueuePlacementRule(String name) { - this.name = name; - } - - public AllocationFileQueuePlacementRule create(boolean create) { - this.create = create; - return this; - } - - public AllocationFileQueuePlacementRule queue(String queue) { - this.queue = queue; - return this; - } - - public AllocationFileQueuePlacementRule addNestedRule( - AllocationFileQueuePlacementRule rule) { - this.nestedRules.add(rule); - return this; - } - - public String render() { - StringWriter sw = new StringWriter(); - PrintWriter pw = new PrintWriter(sw); - addStartTag(pw); - addNestedRules(pw); - addEndTag(pw); - pw.close(); - - return sw.toString(); - } - - private void addStartTag(PrintWriter pw) { - pw.print(""); - } - - private void addNestedRules(PrintWriter pw) { - if (nestedRules != null && !nestedRules.isEmpty()) { - for (AllocationFileQueuePlacementRule rule : nestedRules) { - pw.println(rule.render()); - } - } - } - - private void addEndTag(PrintWriter pw) { - pw.println(""); - } -} diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/allocationfile/AllocationFileQueueProperties.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/allocationfile/AllocationFileQueueProperties.java new file mode 100644 index 00000000000..0a0f330b88c --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/allocationfile/AllocationFileQueueProperties.java @@ -0,0 +1,214 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * http://www.apache.org/licenses/LICENSE-2.0 + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.allocationfile; + +/** + * The purpose of this class is to store all properties of a queue. + */ +public class AllocationFileQueueProperties { + private final String queueName; + private final String minResources; + private final String maxResources; + private final String aclAdministerApps; + private final String aclSubmitApps; + private final String schedulingPolicy; + private final Integer maxRunningApps; + private final Double maxAMShare; + private final Integer minSharePreemptionTimeout; + private final Boolean parent; + private final String maxChildResources; + private final Integer fairSharePreemptionTimeout; + private final Double fairSharePreemptionThreshold; + private final String maxContainerAllocation; + + AllocationFileQueueProperties(Builder builder) { + this.queueName = builder.queueName; + this.parent = builder.parent; + this.minResources = builder.minResources; + this.maxResources = builder.maxResources; + this.aclAdministerApps = builder.aclAdministerApps; + this.aclSubmitApps = builder.aclSubmitApps; + this.schedulingPolicy = builder.schedulingPolicy; + this.maxRunningApps = builder.maxRunningApps; + this.maxAMShare = builder.maxAMShare; + this.minSharePreemptionTimeout = builder.minSharePreemptionTimeout; + this.maxChildResources = builder.maxChildResources; + this.fairSharePreemptionTimeout = builder.fairSharePreemptionTimeout; + this.fairSharePreemptionThreshold = builder.fairSharePreemptionThreshold; + this.maxContainerAllocation = builder.maxContainerAllocation; + } + + public String getQueueName() { + return queueName; + } + + public String getMinResources() { + return minResources; + } + + public String getMaxResources() { + return maxResources; + } + + public String getAclAdministerApps() { + return aclAdministerApps; + } + + public String getAclSubmitApps() { + return aclSubmitApps; + } + + public String getSchedulingPolicy() { + return schedulingPolicy; + } + + public Integer getMaxRunningApps() { + return maxRunningApps; + } + + public Double getMaxAMShare() { + return maxAMShare; + } + + public Integer getMinSharePreemptionTimeout() { + return minSharePreemptionTimeout; + } + + public Boolean getParent() { + return parent; + } + + public String getMaxChildResources() { + return maxChildResources; + } + + public Integer getFairSharePreemptionTimeout() { + return fairSharePreemptionTimeout; + } + + public Double getFairSharePreemptionThreshold() { + return fairSharePreemptionThreshold; + } + + public String getMaxContainerAllocation() { + return maxContainerAllocation; + } + + /** + * Builder class for {@link AllocationFileQueueProperties}. + */ + public static final class Builder { + private String queueName; + private Boolean parent = false; + private String minResources; + private String maxResources; + private String aclAdministerApps; + private String aclSubmitApps; + private String schedulingPolicy; + private Integer maxRunningApps; + private Double maxAMShare; + private Integer minSharePreemptionTimeout; + private String maxChildResources; + private Integer fairSharePreemptionTimeout; + private Double fairSharePreemptionThreshold; + private String maxContainerAllocation; + + Builder() { + } + + public static Builder create() { + return new Builder(); + } + + public Builder queueName(String queueName) { + this.queueName = queueName; + return this; + } + + public Builder minResources(String minResources) { + this.minResources = minResources; + return this; + } + + public Builder maxResources(String maxResources) { + this.maxResources = maxResources; + return this; + } + + public Builder aclAdministerApps(String aclAdministerApps) { + this.aclAdministerApps = aclAdministerApps; + return this; + } + + public Builder aclSubmitApps(String aclSubmitApps) { + this.aclSubmitApps = aclSubmitApps; + return this; + } + + public Builder schedulingPolicy(String schedulingPolicy) { + this.schedulingPolicy = schedulingPolicy; + return this; + } + + public Builder maxRunningApps(Integer maxRunningApps) { + this.maxRunningApps = maxRunningApps; + return this; + } + + public Builder maxAMShare(Double maxAMShare) { + this.maxAMShare = maxAMShare; + return this; + } + + public Builder maxContainerAllocation(String maxContainerAllocation) { + this.maxContainerAllocation = maxContainerAllocation; + return this; + } + + public Builder minSharePreemptionTimeout( + Integer minSharePreemptionTimeout) { + this.minSharePreemptionTimeout = minSharePreemptionTimeout; + return this; + } + + public Builder parent(Boolean parent) { + this.parent = parent; + return this; + } + + public Builder maxChildResources(String maxChildResources) { + this.maxChildResources = maxChildResources; + return this; + } + + public Builder fairSharePreemptionTimeout( + Integer fairSharePreemptionTimeout) { + this.fairSharePreemptionTimeout = fairSharePreemptionTimeout; + return this; + } + + public Builder fairSharePreemptionThreshold( + Double fairSharePreemptionThreshold) { + this.fairSharePreemptionThreshold = fairSharePreemptionThreshold; + return this; + } + + public AllocationFileQueueProperties build() { + return new AllocationFileQueueProperties(this); + } + } +} diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/allocationfile/AllocationFileSimpleQueueBuilder.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/allocationfile/AllocationFileSimpleQueueBuilder.java new file mode 100644 index 00000000000..93d100ed35c --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/allocationfile/AllocationFileSimpleQueueBuilder.java @@ -0,0 +1,64 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * http://www.apache.org/licenses/LICENSE-2.0 + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.allocationfile; + +import java.util.ArrayList; +import java.util.List; + +/** + * Queue builder that can build a simple queue with its properties. + * Subqueues can be added with {@link #addSubQueue(AllocationFileQueue)}. + */ +public class AllocationFileSimpleQueueBuilder + extends AllocationFileQueueBuilder { + private final AllocationFileWriter allocationFileWriter; + private final List subQueues = new ArrayList<>(); + + AllocationFileSimpleQueueBuilder(AllocationFileWriter allocationFileWriter, + String queueName) { + this.allocationFileWriter = allocationFileWriter; + getqueuePropertiesBuilder().queueName(queueName); + } + + void addSubQueue(AllocationFileQueue queue) { + subQueues.add(queue); + } + + @Override + public AllocationFileWriter buildQueue() { + AllocationFileQueueProperties queueProperties = + getqueuePropertiesBuilder().build(); + AllocationFileQueue queue = + new AllocationFileQueue(queueProperties, subQueues); + + if (allocationFileWriter != null) { + allocationFileWriter.addQueue(queue); + } else { + throw new IllegalStateException( + "allocationFileWriter field has to be set on a " + getClass()); + } + + return allocationFileWriter; + } + + @Override + public AllocationFileSimpleQueueBuilder buildSubQueue() { + throw new IllegalStateException( + "buildSubQueue is not supported in " + getClass()); + } + +} diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/allocationfile/AllocationFileSubQueueBuilder.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/allocationfile/AllocationFileSubQueueBuilder.java new file mode 100644 index 00000000000..728aedcbc82 --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/allocationfile/AllocationFileSubQueueBuilder.java @@ -0,0 +1,54 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * http://www.apache.org/licenses/LICENSE-2.0 + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.allocationfile; + +import com.google.common.collect.Lists; + + +/** + * Queue builder that can build a subqueue with its properties. + */ +public class AllocationFileSubQueueBuilder extends AllocationFileQueueBuilder { + private AllocationFileSimpleQueueBuilder parentQueueBuilder; + + AllocationFileSubQueueBuilder( + AllocationFileSimpleQueueBuilder parentQueueBuilder, String queueName) { + getqueuePropertiesBuilder().queueName(queueName); + this.parentQueueBuilder = parentQueueBuilder; + } + + @Override + public AllocationFileWriter buildQueue() { + throw new IllegalStateException( + "BuildQueue is not supported in " + getClass()); + } + + public AllocationFileSimpleQueueBuilder buildSubQueue() { + AllocationFileQueueProperties queueProperties = + getqueuePropertiesBuilder().build(); + AllocationFileQueue queue = + new AllocationFileQueue(queueProperties, Lists.newArrayList()); + + if (parentQueueBuilder != null) { + parentQueueBuilder.addSubQueue(queue); + return parentQueueBuilder; + } else { + throw new IllegalStateException( + "parentQueueBuilder field has to be set on a " + getClass()); + } + } +} diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/allocationfile/AllocationFileWriter.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/allocationfile/AllocationFileWriter.java index f41ca4eeac5..df1cc53d840 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/allocationfile/AllocationFileWriter.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/allocationfile/AllocationFileWriter.java @@ -21,6 +21,7 @@ import java.io.PrintWriter; import java.util.ArrayList; import java.util.List; +import java.util.function.Supplier; /** * This class is capable of serializing allocation file data to a file @@ -38,10 +39,6 @@ private String defaultQueueSchedulingPolicy; private List queues = new ArrayList<>(); private UserSettings userSettings; - private boolean useLegacyTagNameForQueues = false; - private String reservationAgent; - private String reservationPolicy; - private AllocationFileQueuePlacementPolicy queuePlacementPolicy; private AllocationFileWriter() { } @@ -50,9 +47,8 @@ public static AllocationFileWriter create() { return new AllocationFileWriter(); } - public AllocationFileWriter addQueue(AllocationFileQueue queue) { - queues.add(queue); - return this; + public AllocationFileSimpleQueueBuilder queue(String queueName) { + return new AllocationFileSimpleQueueBuilder(this, queueName); } public AllocationFileWriter queueMaxAppsDefault(int value) { @@ -96,41 +92,21 @@ public AllocationFileWriter defaultQueueSchedulingPolicy(String value) { return this; } - public AllocationFileWriter useLegacyTagNameForQueues() { - this.useLegacyTagNameForQueues = true; - return this; - } - - public AllocationFileWriter reservationAgent(String value) { - this.reservationAgent = value; - return this; - } - - public AllocationFileWriter reservationPolicy(String value) { - this.reservationPolicy = value; - return this; + public UserSettings.Builder userSettings(String username) { + return new UserSettings.Builder(this, username); } - public AllocationFileWriter userSettings(UserSettings settings) { - this.userSettings = settings; - return this; + void addQueue(AllocationFileQueue queue) { + this.queues.add(queue); } - public AllocationFileWriter queuePlacementPolicy(AllocationFileQueuePlacementPolicy policy) { - this.queuePlacementPolicy = policy; - return this; + void setUserSettings(UserSettings userSettings) { + this.userSettings = userSettings; } - static void printQueues(PrintWriter pw, List queues, - boolean useLegacyTagName) { + static void printQueues(PrintWriter pw, List queues) { for (AllocationFileQueue queue : queues) { - final String queueStr; - if (useLegacyTagName) { - queueStr = queue.renderWithLegacyTag(); - } else { - queueStr = queue.render(); - } - pw.println(queueStr); + pw.println(queue.render()); } } @@ -138,18 +114,22 @@ private void printUserSettings(PrintWriter pw) { pw.println(userSettings.render()); } - private void printQueuePlacementPolicy(PrintWriter pw) { - pw.println(queuePlacementPolicy.render()); + static void addIfPresent(PrintWriter pw, String tag, + Supplier supplier) { + if (supplier.get() != null) { + pw.println("<" + tag + ">" + supplier.get() + ""); + } } - static void addIfPresent(PrintWriter pw, String tag, Object obj) { - if (obj != null) { - pw.println("<" + tag + ">" + obj.toString() + ""); + static String createNumberSupplier(Object number) { + if (number != null) { + return number.toString(); } + return null; } private void writeHeader(PrintWriter pw) { - pw.println(""); + pw.println(""); pw.println(""); } @@ -160,37 +140,34 @@ private void writeFooter(PrintWriter pw) { public void writeToFile(String filename) { PrintWriter pw; try { - pw = new PrintWriter(filename, "UTF-8"); + pw = new PrintWriter(new FileWriter(filename)); } catch (IOException e) { throw new RuntimeException(e); } writeHeader(pw); if (!queues.isEmpty()) { - printQueues(pw, queues, useLegacyTagNameForQueues); + printQueues(pw, queues); } if (userSettings != null) { printUserSettings(pw); } - if (queuePlacementPolicy != null) { - printQueuePlacementPolicy(pw); - } - - addIfPresent(pw, "queueMaxAppsDefault", queueMaxAppsDefault); - addIfPresent(pw, "queueMaxResourcesDefault", queueMaxResourcesDefault); - addIfPresent(pw, "userMaxAppsDefault", userMaxAppsDefault); - addIfPresent(pw, "queueMaxAMShareDefault", queueMaxAMShareDefault); + addIfPresent(pw, "queueMaxAppsDefault", + () -> createNumberSupplier(queueMaxAppsDefault)); + addIfPresent(pw, "queueMaxResourcesDefault", + () -> queueMaxResourcesDefault); + addIfPresent(pw, "userMaxAppsDefault", + () -> createNumberSupplier(userMaxAppsDefault)); + addIfPresent(pw, "queueMaxAMShareDefault", + () -> createNumberSupplier(queueMaxAMShareDefault)); addIfPresent(pw, "defaultMinSharePreemptionTimeout", - defaultMinSharePreemptionTimeout); + () -> createNumberSupplier(defaultMinSharePreemptionTimeout)); addIfPresent(pw, "defaultFairSharePreemptionTimeout", - defaultFairSharePreemptionTimeout); + () -> createNumberSupplier(defaultFairSharePreemptionTimeout)); addIfPresent(pw, "defaultFairSharePreemptionThreshold", - defaultFairSharePreemptionThreshold); + () -> createNumberSupplier(defaultFairSharePreemptionThreshold)); addIfPresent(pw, "defaultQueueSchedulingPolicy", - defaultQueueSchedulingPolicy); - addIfPresent(pw, "reservation-agent", reservationAgent); - addIfPresent(pw, "reservation-policy", reservationPolicy); - + () -> defaultQueueSchedulingPolicy); writeFooter(pw); pw.close(); } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/allocationfile/UserSettings.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/allocationfile/UserSettings.java index e8457d0e8ec..7a5656e5f3a 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/allocationfile/UserSettings.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/allocationfile/UserSettings.java @@ -23,7 +23,7 @@ * Value class that stores user settings and can render data in XML format, * see {@link #render()}. */ -public class UserSettings { +class UserSettings { private final String username; private final Integer maxRunningApps; @@ -36,7 +36,8 @@ public String render() { StringWriter sw = new StringWriter(); PrintWriter pw = new PrintWriter(sw); addStartTag(pw); - AllocationFileWriter.addIfPresent(pw, "maxRunningApps", maxRunningApps); + AllocationFileWriter.addIfPresent(pw, "maxRunningApps", + () -> AllocationFileWriter.createNumberSupplier(maxRunningApps)); addEndTag(pw); pw.close(); @@ -55,10 +56,12 @@ private void addEndTag(PrintWriter pw) { * Builder class for {@link UserSettings} */ public static class Builder { + private final AllocationFileWriter allocationFileWriter; private final String username; private Integer maxRunningApps; - public Builder(String username) { + Builder(AllocationFileWriter allocationFileWriter, String username) { + this.allocationFileWriter = allocationFileWriter; this.username = username; } @@ -67,8 +70,11 @@ public Builder maxRunningApps(int value) { return this; } - public UserSettings build() { - return new UserSettings(this); + public AllocationFileWriter build() { + UserSettings userSettings = new UserSettings(this); + allocationFileWriter.setUserSettings(userSettings); + + return allocationFileWriter; } } } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesAppsModification.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesAppsModification.java index d370a5951a5..5d99e0d169d 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesAppsModification.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesAppsModification.java @@ -81,11 +81,6 @@ import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacitySchedulerConfiguration; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.FairScheduler; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.FairSchedulerConfiguration; - -import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair - .allocationfile.AllocationFileQueue; -import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair - .allocationfile.AllocationFileWriter; import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.AppPriority; import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.AppQueue; import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.AppState; @@ -207,15 +202,23 @@ public void configureScheduler() { private class FairTestServletModule extends TestServletModule { @Override public void configureScheduler() { - AllocationFileWriter.create() - .addQueue(new AllocationFileQueue.Builder("root") - .aclAdministerApps("someuser ") - .subQueue(new AllocationFileQueue.Builder("default") - .aclAdministerApps("someuser ").build()) - .subQueue(new AllocationFileQueue.Builder("test") - .aclAdministerApps("someuser ").build()) - .build()) - .writeToFile(FS_ALLOC_FILE); + try { + PrintWriter out = new PrintWriter(new FileWriter(FS_ALLOC_FILE)); + out.println(""); + out.println(""); + out.println(""); + out.println(" someuser "); + out.println(" "); + out.println(" someuser "); + out.println(" "); + out.println(" "); + out.println(" someuser "); + out.println(" "); + out.println(""); + out.println(""); + out.close(); + } catch(IOException e) { + } conf.set(FairSchedulerConfiguration.ALLOCATION_FILE, FS_ALLOC_FILE); conf.set(YarnConfiguration.RM_SCHEDULER, FairScheduler.class.getName()); } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesReservation.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesReservation.java index 57795d7466c..adf6d7cb36c 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesReservation.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesReservation.java @@ -53,11 +53,6 @@ import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacitySchedulerConfiguration; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.FairScheduler; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.FairSchedulerConfiguration; - -import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair - .allocationfile.AllocationFileQueue; -import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair - .allocationfile.AllocationFileWriter; import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.ReservationDeleteRequestInfo; import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.ReservationSubmissionRequestInfo; import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.ReservationUpdateRequestInfo; @@ -186,17 +181,27 @@ public void configureScheduler() { private static class FairTestServletModule extends TestServletModule { @Override public void configureScheduler() { - AllocationFileWriter.create() - .defaultQueueSchedulingPolicy("drf") - .addQueue(new AllocationFileQueue.Builder("root") - .aclAdministerApps("someuser ") - .subQueue(new AllocationFileQueue.Builder("default") - .aclAdministerApps("someuser ").build()) - .subQueue(new AllocationFileQueue.Builder("dedicated") - .reservation() - .aclAdministerApps("someuser ").build()) - .build()) - .writeToFile(FS_ALLOC_FILE); + try { + PrintWriter out = new PrintWriter(new FileWriter(FS_ALLOC_FILE)); + out.println(""); + out.println(""); + out.println(""); + out.println(" someuser "); + out.println(" "); + out.println(" someuser "); + out.println(" "); + out.println(" "); + out.println(" "); + out.println(" "); + out.println(" someuser "); + out.println(" "); + out.println(""); + out.println("drf" + + ""); + out.println(""); + out.close(); + } catch (IOException e) { + } conf.set(FairSchedulerConfiguration.ALLOCATION_FILE, FS_ALLOC_FILE); conf.set(YarnConfiguration.RM_SCHEDULER, FairScheduler.class.getName()); }