diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/api/impl/TestYarnClientWithReservation.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/api/impl/TestYarnClientWithReservation.java index 0836b7b9c97..81f6ac06808 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/api/impl/TestYarnClientWithReservation.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/api/impl/TestYarnClientWithReservation.java @@ -45,6 +45,11 @@ import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacitySchedulerConfiguration; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.FairScheduler; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.FairSchedulerConfiguration; + +import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair + .allocationfile.AllocationFileQueue; +import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair + .allocationfile.AllocationFileWriter; import org.apache.hadoop.yarn.util.Clock; import org.apache.hadoop.yarn.util.UTCClock; import org.junit.Assert; @@ -134,25 +139,16 @@ private Configuration configureReservationForCapacityScheduler() { private Configuration configureReservationForFairScheduler() { Configuration conf = new Configuration(); - try { - PrintWriter out = new PrintWriter(new FileWriter(FS_ALLOC_FILE)); - out.println(""); - out.println(""); - out.println(""); - out.println(" "); - out.println(" "); - out.println(" "); - // set weight to 10 to make sure this queue get enough steady fair share - out.println(" 10"); - out.println(" "); - out.println(""); - out.println("drf" + - ""); - out.println(""); - out.close(); - } catch (IOException e) { - Assert.fail(e.getMessage()); - } + AllocationFileWriter.create() + .defaultQueueSchedulingPolicy("drf") + .addQueue(new AllocationFileQueue.Builder("root") + .subQueue(new AllocationFileQueue.Builder("default").build()) + .subQueue(new AllocationFileQueue.Builder("dedicated") + .reservation() + .weight(10) + .build()) + .build()) + .writeToFile(FS_ALLOC_FILE); conf.set(FairSchedulerConfiguration.ALLOCATION_FILE, FS_ALLOC_FILE); return conf; diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/ParameterizedSchedulerTestBase.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/ParameterizedSchedulerTestBase.java index 4de16dc88b0..f4193fb6cc0 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/ParameterizedSchedulerTestBase.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/ParameterizedSchedulerTestBase.java @@ -26,6 +26,12 @@ import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.FairScheduler; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.FairSchedulerConfiguration; + + +import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair + .allocationfile.AllocationFileQueue; +import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair + .allocationfile.AllocationFileWriter; import org.junit.After; import org.junit.runner.RunWith; import org.junit.runners.Parameterized; @@ -93,20 +99,17 @@ public ParameterizedSchedulerTestBase(SchedulerType type) protected void configureFairScheduler(YarnConfiguration conf) throws IOException { // Disable queueMaxAMShare limitation for fair scheduler - PrintWriter out = new PrintWriter(new FileWriter(FS_ALLOC_FILE)); - out.println(""); - out.println(""); - out.println("-1.0"); - out.println("fair"); - out.println(""); - out.println(" drf"); - out.println(" 1.0"); - out.println(" 100"); - out.println(" 120"); - out.println(" .5"); - out.println(""); - out.println(""); - out.close(); + AllocationFileWriter.create() + .defaultQueueSchedulingPolicy("fair") + .queueMaxAMShareDefault(-1.0) + .addQueue(new AllocationFileQueue.Builder("root") + .schedulingPolicy("drf") + .weight(1.0f) + .fairSharePreemptionTimeout(100) + .minSharePreemptionTimeout(120) + .fairSharePreemptionThreshold(.5) + .build()) + .writeToFile(FS_ALLOC_FILE); conf.set(FairSchedulerConfiguration.ALLOCATION_FILE, FS_ALLOC_FILE); conf.setLong(FairSchedulerConfiguration.UPDATE_INTERVAL_MS, 10); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/ReservationACLsTestBase.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/ReservationACLsTestBase.java index c8ee00e60bc..e9ba98ea07f 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/ReservationACLsTestBase.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/ReservationACLsTestBase.java @@ -53,6 +53,12 @@ import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacitySchedulerConfiguration; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.FairScheduler; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.FairSchedulerConfiguration; + + +import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair + .allocationfile.AllocationFileQueue; +import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair + .allocationfile.AllocationFileWriter; import org.apache.hadoop.yarn.server.utils.BuilderUtils; import org.junit.After; import org.junit.Assert; @@ -552,48 +558,34 @@ private static Configuration createCapacitySchedulerConfiguration() { return csConf; } - private static Configuration createFairSchedulerConfiguration() throws - IOException { + private static Configuration createFairSchedulerConfiguration() { FairSchedulerConfiguration fsConf = new FairSchedulerConfiguration(); final String TEST_DIR = new File(System.getProperty("test.build.data", "/tmp")).getAbsolutePath(); final String ALLOC_FILE = new File(TEST_DIR, "test-queues.xml") .getAbsolutePath(); - PrintWriter out = new PrintWriter(new FileWriter(ALLOC_FILE)); - out.println(""); - out.println(""); - out.println(" drf" + - ""); - out.println(" "); - out.println(" " + - "queueA_user,common_user " + - ""); - out.println(" " + - "queueA_admin " + - ""); - out.println(" common_user "); - out.println(" queueA_user,common_user "); - out.println(" queueA_admin "); - out.println(" "); - out.println(" "); - out.println(" "); - out.println(" queueB_user,common_user "); - out.println(" queueB_admin "); - out.println(" " + - "queueB_user,common_user " + - ""); - out.println(" " + - "queueB_admin " + - ""); - out.println(" common_user "); - out.println(" "); - out.println(" "); - out.println(" "); - out.println(" "); - out.println(" "); - out.println(""); - out.close(); + + AllocationFileWriter.create() + .defaultQueueSchedulingPolicy("drf") + .addQueue(new AllocationFileQueue.Builder("queueA") + .aclSubmitReservations("queueA_user,common_user ") + .aclAdministerReservations("queueA_admin ") + .aclListReservations("common_user ") + .aclSubmitApps("queueA_user,common_user ") + .aclAdministerApps("queueA_admin ") + .reservation().build()) + .addQueue(new AllocationFileQueue.Builder("queueB") + .aclSubmitReservations("queueB_user,common_user ") + .aclAdministerReservations("queueB_admin ") + .aclListReservations("common_user ") + .aclSubmitApps("queueB_user,common_user ") + .aclAdministerApps("queueB_admin ") + .reservation().build()) + .addQueue(new AllocationFileQueue.Builder("queueC") + .reservation().build()) + .writeToFile(ALLOC_FILE); + fsConf.set(FairSchedulerConfiguration.ALLOCATION_FILE, ALLOC_FILE); fsConf.setBoolean(YarnConfiguration.RM_RESERVATION_SYSTEM_ENABLE, true); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestAppManagerWithFairScheduler.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestAppManagerWithFairScheduler.java index feb7ed246ce..f44a9d79d73 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestAppManagerWithFairScheduler.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestAppManagerWithFairScheduler.java @@ -49,6 +49,12 @@ import org.apache.hadoop.yarn.server.resourcemanager.scheduler.ResourceScheduler; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.FairScheduler; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.FairSchedulerConfiguration; + + +import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair + .allocationfile.AllocationFileQueue; +import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair + .allocationfile.AllocationFileWriter; import org.apache.hadoop.yarn.server.resourcemanager.security.ClientToAMTokenSecretManagerInRM; import org.apache.hadoop.yarn.server.security.ApplicationACLsManager; import org.apache.hadoop.yarn.util.resource.Resources; @@ -70,23 +76,18 @@ private static YarnConfiguration conf = new YarnConfiguration(); @BeforeClass - public static void setup() throws IOException { + public static void setup() { String allocFile = GenericTestUtils.getTestDir(TEST_FOLDER).getAbsolutePath(); int queueMaxAllocation = 512; - PrintWriter out = new PrintWriter(new FileWriter(allocFile)); - out.println(""); - out.println(""); - out.println(" "); - out.println(" " + queueMaxAllocation - + " mb 1 vcores" + ""); - out.println(" "); - out.println(" "); - out.println(" "); - out.println(""); - out.close(); + AllocationFileWriter.create() + .addQueue(new AllocationFileQueue.Builder("queueA") + .maxContainerAllocation(queueMaxAllocation + " mb 1 vcores") + .build()) + .addQueue(new AllocationFileQueue.Builder("queueB").build()) + .writeToFile(allocFile); conf.setClass(YarnConfiguration.RM_SCHEDULER, FairScheduler.class, ResourceScheduler.class); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestApplicationACLs.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestApplicationACLs.java index c7ed02c9c29..168a30e61c2 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestApplicationACLs.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestApplicationACLs.java @@ -35,6 +35,16 @@ import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMApp; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.FairScheduler; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.FairSchedulerConfiguration; + + +import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair + .allocationfile.AllocationFileQueue; +import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair + .allocationfile.AllocationFileQueuePlacementPolicy; +import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair + .allocationfile.AllocationFileQueuePlacementRule; +import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair + .allocationfile.AllocationFileWriter; import org.junit.After; import org.junit.Assert; import org.apache.commons.logging.Log; @@ -180,25 +190,22 @@ public void tearDown() { } @Override - protected void configureFairScheduler(YarnConfiguration conf) - throws IOException { + protected void configureFairScheduler(YarnConfiguration conf) { final String testDir = new File(System.getProperty("test.build.data", "/tmp")).getAbsolutePath(); final String allocFile = new File(testDir, "test-queues.xml") .getAbsolutePath(); - PrintWriter out = new PrintWriter(new FileWriter(allocFile)); - out.println(""); - out.println(""); - out.println(""); - out.println(" "); - out.println(" "); - out.println(""); - out.println(""); - out.println(" "); - out.println(" "); - out.println(""); - out.println(""); - out.close(); + + AllocationFileWriter.create() + .addQueue(new AllocationFileQueue.Builder("root") + .subQueue(new AllocationFileQueue.Builder("default").build()) + .build()) + .queuePlacementPolicy(new AllocationFileQueuePlacementPolicy() + .addRule(new AllocationFileQueuePlacementRule("specified") + .create(false)) + .addRule(new AllocationFileQueuePlacementRule("reject"))) + .writeToFile(allocFile); + conf.set(FairSchedulerConfiguration.ALLOCATION_FILE, allocFile); } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/reservation/ReservationSystemTestUtil.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/reservation/ReservationSystemTestUtil.java index 09d3327263b..67676d1fcf0 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/reservation/ReservationSystemTestUtil.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/reservation/ReservationSystemTestUtil.java @@ -56,6 +56,11 @@ import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacitySchedulerConfiguration; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.event.NodeAddedSchedulerEvent; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.FairScheduler; + +import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair + .allocationfile.AllocationFileQueue; +import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair + .allocationfile.AllocationFileWriter; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.placement.MultiNodeSortingManager; import org.apache.hadoop.yarn.server.resourcemanager.security.ClientToAMTokenSecretManagerInRM; import org.apache.hadoop.yarn.server.resourcemanager.security.NMTokenSecretManagerInRM; @@ -104,62 +109,46 @@ public static void validateReservationQueue( .assertTrue(plan.getSharingPolicy() instanceof CapacityOverTimePolicy); } - public static void setupFSAllocationFile(String allocationFile) - throws IOException { - PrintWriter out = new PrintWriter(new FileWriter(allocationFile)); - out.println(""); - out.println(""); - out.println(""); - out.println("1"); - out.println(""); - out.println(""); - out.println("1"); - out.println(""); - out.println("3"); - out.println(""); - out.println(""); - out.println("7"); - out.println(""); - out.println(""); - out.println(""); - out.println(""); - out.println("8"); - out.println(""); - out.println( - "drf"); - out.println(""); - out.close(); + public static void setupFSAllocationFile(String allocationFile) { + AllocationFileWriter.create() + .defaultQueueSchedulingPolicy("drf") + .addQueue(new AllocationFileQueue.Builder("default") + .weight(1).build()) + .addQueue(new AllocationFileQueue.Builder("a") + .weight(1) + .subQueue(new AllocationFileQueue.Builder("a1") + .weight(3).build()) + .subQueue(new AllocationFileQueue.Builder("a2") + .weight(7).build()) + .build()) + .addQueue(new AllocationFileQueue.Builder("dedicated") + .weight(8) + .reservation() + .build()) + .writeToFile(allocationFile); } - public static void updateFSAllocationFile(String allocationFile) - throws IOException { - PrintWriter out = new PrintWriter(new FileWriter(allocationFile)); - out.println(""); - out.println(""); - out.println(""); - out.println("5"); - out.println(""); - out.println(""); - out.println("5"); - out.println(""); - out.println("3"); - out.println(""); - out.println(""); - out.println("7"); - out.println(""); - out.println(""); - out.println(""); - out.println(""); - out.println("10"); - out.println(""); - out.println(""); - out.println(""); - out.println("80"); - out.println(""); - out.println( - "drf"); - out.println(""); - out.close(); + public static void updateFSAllocationFile(String allocationFile) { + AllocationFileWriter.create() + .defaultQueueSchedulingPolicy("drf") + .addQueue(new AllocationFileQueue.Builder("default") + .weight(5).build()) + .addQueue(new AllocationFileQueue.Builder("a") + .weight(5) + .subQueue(new AllocationFileQueue.Builder("a1") + .weight(3).build()) + .subQueue(new AllocationFileQueue.Builder("a2") + .weight(7).build()) + .build()) + .addQueue(new AllocationFileQueue.Builder("dedicated") + .weight(10) + .reservation() + .build()) + .addQueue(new AllocationFileQueue.Builder("reservation") + .weight(80) + .reservation() + .build()) + .writeToFile(allocationFile); } public static FairScheduler setupFairScheduler(RMContext rmContext, diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestAllocationFileLoaderService.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestAllocationFileLoaderService.java index ac30b237472..7f875439cee 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestAllocationFileLoaderService.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestAllocationFileLoaderService.java @@ -27,24 +27,27 @@ import org.apache.hadoop.yarn.conf.YarnConfiguration; import org.apache.hadoop.yarn.server.resourcemanager.reservation.ReservationSchedulerConfiguration; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.QueuePlacementRule.NestedUserQueue; +import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.allocationfile.AllocationFileQueue; +import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.allocationfile.AllocationFileQueuePlacementPolicy; +import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.allocationfile.AllocationFileQueuePlacementRule; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.allocationfile.AllocationFileWriter; +import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.allocationfile.UserSettings; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.policies.DominantResourceFairnessPolicy; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.policies.FairSharePolicy; import org.apache.hadoop.yarn.util.ControlledClock; -import org.apache.hadoop.yarn.util.resource.ResourceUtils; import org.apache.hadoop.yarn.util.resource.Resources; import org.apache.hadoop.yarn.util.resource.TestResourceUtils; +import org.junit.After; import org.junit.Test; + import java.io.File; import java.io.FileOutputStream; -import java.io.FileWriter; import java.io.IOException; import java.io.OutputStreamWriter; import java.io.PrintWriter; import java.net.URISyntaxException; import java.net.URL; import java.nio.charset.StandardCharsets; -import java.util.HashMap; import java.util.List; import java.util.Map; import java.util.Set; @@ -122,18 +125,13 @@ public void testGetAllocationFileFromClasspath() { @Test (timeout = 10000) public void testReload() throws Exception { - PrintWriter out = new PrintWriter(new FileWriter(ALLOC_FILE)); - out.println(""); - out.println(""); - out.println(" "); - out.println(" 1"); - out.println(" "); - out.println(" "); - out.println(" "); - out.println(" "); - out.println(" "); - out.println(""); - out.close(); + AllocationFileWriter.create() + .addQueue(new AllocationFileQueue.Builder("queueA") + .maxRunningApps(1).build()) + .addQueue(new AllocationFileQueue.Builder("queueB").build()) + .queuePlacementPolicy(new AllocationFileQueuePlacementPolicy() + .addRule(new AllocationFileQueuePlacementRule("default"))) + .writeToFile(ALLOC_FILE); ControlledClock clock = new ControlledClock(); clock.setTime(0); @@ -165,21 +163,15 @@ public void testReload() throws Exception { confHolder.allocConf = null; // Modify file and advance the clock - out = new PrintWriter(new FileWriter(ALLOC_FILE)); - out.println(""); - out.println(""); - out.println(" "); - out.println(" 3"); - out.println(" "); - out.println(" "); - out.println(" "); - out.println(" "); - out.println(" "); - out.println(" "); - out.println(" "); - out.println(" "); - out.println(""); - out.close(); + AllocationFileWriter.create() + .addQueue(new AllocationFileQueue.Builder("queueB") + .maxRunningApps(3).build()) + .queuePlacementPolicy(new AllocationFileQueuePlacementPolicy() + .addRule(new AllocationFileQueuePlacementRule("specified")) + .addRule(new AllocationFileQueuePlacementRule("nestedUserQueue") + .addNestedRule(new AllocationFileQueuePlacementRule("primaryGroup"))) + .addRule(new AllocationFileQueuePlacementRule("default"))) + .writeToFile(ALLOC_FILE); clock.tickMsec(System.currentTimeMillis() + AllocationFileLoaderService.ALLOC_RELOAD_WAIT_MS + 10000); @@ -214,57 +206,56 @@ public void testAllocationFileParsing() throws Exception { conf.set(FairSchedulerConfiguration.ALLOCATION_FILE, ALLOC_FILE); AllocationFileLoaderService allocLoader = new AllocationFileLoaderService(); - AllocationFileWriter - .create() + AllocationFileWriter.create() // Give queue A a minimum of 1024 M - .queue("queueA") - .minResources("1024mb,0vcores") - .maxResources("2048mb,10vcores") - .buildQueue() + .addQueue(new AllocationFileQueue.Builder("queueA") + .minResources("1024mb,0vcores") + .maxResources("2048mb,10vcores") + .build()) // Give queue B a minimum of 2048 M - .queue("queueB") + .addQueue(new AllocationFileQueue.Builder("queueB") .minResources("2048mb,0vcores") .maxResources("5120mb,110vcores") .aclAdministerApps("alice,bob admins") .schedulingPolicy("fair") - .buildQueue() + .build()) // Give queue C no minimum - .queue("queueC") - .minResources("5120mb,0vcores") - .aclSubmitApps("alice,bob admins") - .buildQueue() + .addQueue(new AllocationFileQueue.Builder("queueC") + .minResources("5120mb,0vcores") + .aclSubmitApps("alice,bob admins") + .build()) // Give queue D a limit of 3 running apps and 0.4f maxAMShare - .queue("queueD") - .maxRunningApps(3) - .maxAMShare(0.4) - .buildQueue() + .addQueue(new AllocationFileQueue.Builder("queueD") + .maxRunningApps(3) + .maxAMShare(0.4) + .build()) // Give queue E a preemption timeout of one minute - .queue("queueE") - .minSharePreemptionTimeout(60) - .buildQueue() + .addQueue(new AllocationFileQueue.Builder("queueE") + .minSharePreemptionTimeout(60) + .build()) // Make queue F a parent queue without configured leaf queues // using the 'type' attribute - .queue("queueF") - .parent(true) - .maxChildResources("2048mb,64vcores") - .buildQueue() - .queue("queueG") - .maxChildResources("2048mb,64vcores") - .fairSharePreemptionTimeout(120) - .minSharePreemptionTimeout(50) - .fairSharePreemptionThreshold(0.6) - .maxContainerAllocation( + .addQueue(new AllocationFileQueue.Builder("queueF") + .parent(true) + .maxChildResources("2048mb,64vcores") + .build()) + .addQueue(new AllocationFileQueue.Builder("queueG") + .maxChildResources("2048mb,64vcores") + .fairSharePreemptionTimeout(120) + .minSharePreemptionTimeout(50) + .fairSharePreemptionThreshold(0.6) + .maxContainerAllocation( "vcores=16, memory-mb=512, " + A_CUSTOM_RESOURCE + "=10") // Create hierarchical queues G,H, with different min/fair // share preemption timeouts and preemption thresholds. // Also add a child default to make sure it doesn't impact queue H. - .subQueue("queueH") - .fairSharePreemptionTimeout(180) - .minSharePreemptionTimeout(40) - .fairSharePreemptionThreshold(0.7) - .maxContainerAllocation("1024mb,8vcores") - .buildSubQueue() - .buildQueue() + .subQueue(new AllocationFileQueue.Builder("queueH") + .fairSharePreemptionTimeout(180) + .minSharePreemptionTimeout(40) + .fairSharePreemptionThreshold(0.7) + .maxContainerAllocation("1024mb,8vcores") + .build()) + .build()) // Set default limit of apps per queue to 15 .queueMaxAppsDefault(15) // Set default limit of max resource per queue to 4G and 100 cores @@ -282,9 +273,9 @@ public void testAllocationFileParsing() throws Exception { // Set default scheduling policy to DRF .defaultQueueSchedulingPolicy("drf") // Give user1 a limit of 10 jobs - .userSettings("user1") + .userSettings(new UserSettings.Builder("user1") .maxRunningApps(10) - .build() + .build()) .writeToFile(ALLOC_FILE); allocLoader.init(conf); @@ -460,48 +451,51 @@ public void testBackwardsCompatibleAllocationFileParsing() throws Exception { conf.set(FairSchedulerConfiguration.ALLOCATION_FILE, ALLOC_FILE); AllocationFileLoaderService allocLoader = new AllocationFileLoaderService(); - PrintWriter out = new PrintWriter(new FileWriter(ALLOC_FILE)); - out.println(""); - out.println(""); - // Give queue A a minimum of 1024 M - out.println(""); - out.println("1024mb,0vcores"); - out.println(""); - // Give queue B a minimum of 2048 M - out.println(""); - out.println("2048mb,0vcores"); - out.println("alice,bob admins"); - out.println(""); - // Give queue C no minimum - out.println(""); - out.println("alice,bob admins"); - out.println(""); - // Give queue D a limit of 3 running apps - out.println(""); - out.println("3"); - out.println(""); - // Give queue E a preemption timeout of one minute and 0.3f threshold - out.println(""); - out.println("60"); - out.println("0.3"); - out.println(""); - // Set default limit of apps per queue to 15 - out.println("15"); - // Set default limit of apps per user to 5 - out.println("5"); - // Give user1 a limit of 10 jobs - out.println(""); - out.println("10"); - out.println(""); - // Set default min share preemption timeout to 2 minutes - out.println("120" - + ""); - // Set fair share preemption timeout to 5 minutes - out.println("300"); - // Set default fair share preemption threshold to 0.6f - out.println("0.6"); - out.println(""); - out.close(); + AllocationFileWriter.create() + .useLegacyTagNameForQueues() + // Give queue A a minimum of 1024 M + .addQueue(new AllocationFileQueue.Builder("queueA") + .minResources("1024mb,0vcores") + .build()) + // Give queue B a minimum of 2048 M + .addQueue(new AllocationFileQueue.Builder("queueB") + .minResources("2048mb,0vcores") + .aclAdministerApps("alice,bob admins") + .build()) + // Give queue C no minimum + .addQueue(new AllocationFileQueue.Builder("queueC") + .aclAdministerApps("alice,bob admins") + .build()) + // Give queue D a limit of 3 running apps + .addQueue(new AllocationFileQueue.Builder("queueD") + .maxRunningApps(3) + .build()) + // Give queue E a preemption timeout of one minute and 0.3f threshold + .addQueue(new AllocationFileQueue.Builder("queueE") + .minSharePreemptionTimeout(60) + .fairSharePreemptionThreshold(0.3) + .build()) + // Set default limit of apps per queue to 15 + .queueMaxAppsDefault(15) + // Set default limit of apps per user to 5 + .userMaxAppsDefault(5) + // Set default limit of max resource per queue to 4G and 100 cores + .queueMaxResourcesDefault("4096mb,100vcores") + // Set default limit of AMResourceShare to 0.5f + .queueMaxAMShareDefault(0.5) + // Set default min share preemption timeout to 2 minutes + .defaultMinSharePreemptionTimeout(120) + // Set default fair share preemption timeout to 5 minutes + .defaultFairSharePreemptionTimeout(300) + // Set default fair share preemption threshold to 0.6 + .defaultFairSharePreemptionThreshold(0.6) + // Set default scheduling policy to DRF + .defaultQueueSchedulingPolicy("drf") + // Give user1 a limit of 10 jobs + .userSettings(new UserSettings.Builder("user1") + .maxRunningApps(10) + .build()) + .writeToFile(ALLOC_FILE); allocLoader.init(conf); ReloadListener confHolder = new ReloadListener(); @@ -575,11 +569,7 @@ public void testSimplePlacementPolicyFromConf() throws Exception { conf.setBoolean(FairSchedulerConfiguration.ALLOW_UNDECLARED_POOLS, false); conf.setBoolean(FairSchedulerConfiguration.USER_AS_DEFAULT_QUEUE, false); - PrintWriter out = new PrintWriter(new FileWriter(ALLOC_FILE)); - out.println(""); - out.println(""); - out.println(""); - out.close(); + AllocationFileWriter.create().writeToFile(ALLOC_FILE); AllocationFileLoaderService allocLoader = new AllocationFileLoaderService(); allocLoader.init(conf); @@ -605,15 +595,10 @@ public void testQueueAlongsideRoot() throws Exception { Configuration conf = new Configuration(); conf.set(FairSchedulerConfiguration.ALLOCATION_FILE, ALLOC_FILE); - PrintWriter out = new PrintWriter(new FileWriter(ALLOC_FILE)); - out.println(""); - out.println(""); - out.println(""); - out.println(""); - out.println(""); - out.println(""); - out.println(""); - out.close(); + AllocationFileWriter.create() + .addQueue(new AllocationFileQueue.Builder("root").build()) + .addQueue(new AllocationFileQueue.Builder("other").build()) + .writeToFile(ALLOC_FILE); AllocationFileLoaderService allocLoader = new AllocationFileLoaderService(); allocLoader.init(conf); @@ -631,13 +616,9 @@ public void testQueueNameContainingPeriods() throws Exception { Configuration conf = new Configuration(); conf.set(FairSchedulerConfiguration.ALLOCATION_FILE, ALLOC_FILE); - PrintWriter out = new PrintWriter(new FileWriter(ALLOC_FILE)); - out.println(""); - out.println(""); - out.println(""); - out.println(""); - out.println(""); - out.close(); + AllocationFileWriter.create() + .addQueue(new AllocationFileQueue.Builder("parent1.child").build()) + .writeToFile(ALLOC_FILE); AllocationFileLoaderService allocLoader = new AllocationFileLoaderService(); allocLoader.init(conf); @@ -655,13 +636,9 @@ public void testQueueNameContainingOnlyWhitespace() throws Exception { Configuration conf = new Configuration(); conf.set(FairSchedulerConfiguration.ALLOCATION_FILE, ALLOC_FILE); - PrintWriter out = new PrintWriter(new FileWriter(ALLOC_FILE)); - out.println(""); - out.println(""); - out.println(""); - out.println(""); - out.println(""); - out.close(); + AllocationFileWriter.create() + .addQueue(new AllocationFileQueue.Builder(" ").build()) + .writeToFile(ALLOC_FILE); AllocationFileLoaderService allocLoader = new AllocationFileLoaderService(); allocLoader.init(conf); @@ -675,15 +652,12 @@ public void testParentTagWithReservation() throws Exception { Configuration conf = new Configuration(); conf.set(FairSchedulerConfiguration.ALLOCATION_FILE, ALLOC_FILE); - PrintWriter out = new PrintWriter(new FileWriter(ALLOC_FILE)); - out.println(""); - out.println(""); - out.println(""); - out.println(""); - out.println(""); - out.println(""); - out.println(""); - out.close(); + AllocationFileWriter.create() + .addQueue(new AllocationFileQueue.Builder("parent") + .parent(true) + .reservation() + .build()) + .writeToFile(ALLOC_FILE); AllocationFileLoaderService allocLoader = new AllocationFileLoaderService(); allocLoader.init(conf); @@ -704,17 +678,13 @@ public void testParentWithReservation() throws Exception { Configuration conf = new Configuration(); conf.set(FairSchedulerConfiguration.ALLOCATION_FILE, ALLOC_FILE); - PrintWriter out = new PrintWriter(new FileWriter(ALLOC_FILE)); - out.println(""); - out.println(""); - out.println(""); - out.println(""); - out.println(""); - out.println(" "); - out.println(" "); - out.println(""); - out.println(""); - out.close(); + AllocationFileWriter.create() + .addQueue(new AllocationFileQueue.Builder("parent") + .parent(true) + .subQueue(new AllocationFileQueue.Builder("child").build()) + .reservation() + .build()) + .writeToFile(ALLOC_FILE); AllocationFileLoaderService allocLoader = new AllocationFileLoaderService(); allocLoader.init(conf); @@ -735,15 +705,12 @@ public void testParentTagWithChild() throws Exception { Configuration conf = new Configuration(); conf.set(FairSchedulerConfiguration.ALLOCATION_FILE, ALLOC_FILE); - PrintWriter out = new PrintWriter(new FileWriter(ALLOC_FILE)); - out.println(""); - out.println(""); - out.println(""); - out.println(" "); - out.println(" "); - out.println(""); - out.println(""); - out.close(); + AllocationFileWriter.create() + .addQueue(new AllocationFileQueue.Builder("parent") + .parent(true) + .subQueue(new AllocationFileQueue.Builder("child").build()) + .build()) + .writeToFile(ALLOC_FILE); AllocationFileLoaderService allocLoader = new AllocationFileLoaderService(); allocLoader.init(conf); @@ -767,6 +734,22 @@ public void testQueueNameContainingNBWhitespace() throws Exception { Configuration conf = new Configuration(); conf.set(FairSchedulerConfiguration.ALLOCATION_FILE, ALLOC_FILE); + AllocationFileWriter.create() + .addQueue(new AllocationFileQueue.Builder("\u00a0").build()) + .writeToFile(ALLOC_FILE); + + AllocationFileLoaderService allocLoader = new AllocationFileLoaderService(); + allocLoader.init(conf); + ReloadListener confHolder = new ReloadListener(); + allocLoader.setReloadListener(confHolder); + allocLoader.reloadAllocations(); + } + + @Test (expected = AllocationConfigurationException.class) + public void testQueueNameContainingNBWhitespace2() throws Exception { + Configuration conf = new Configuration(); + conf.set(FairSchedulerConfiguration.ALLOCATION_FILE, ALLOC_FILE); + PrintWriter out = new PrintWriter(new OutputStreamWriter( new FileOutputStream(ALLOC_FILE), StandardCharsets.UTF_8)); out.println(""); @@ -783,6 +766,7 @@ public void testQueueNameContainingNBWhitespace() throws Exception { allocLoader.reloadAllocations(); } + /** * Verify that defaultQueueSchedulingMode can't accept FIFO as a value. */ @@ -791,12 +775,9 @@ public void testDefaultQueueSchedulingModeIsFIFO() throws Exception { Configuration conf = new Configuration(); conf.set(FairSchedulerConfiguration.ALLOCATION_FILE, ALLOC_FILE); - PrintWriter out = new PrintWriter(new FileWriter(ALLOC_FILE)); - out.println(""); - out.println(""); - out.println("fifo"); - out.println(""); - out.close(); + AllocationFileWriter.create() + .defaultQueueSchedulingPolicy("fifo") + .writeToFile(ALLOC_FILE); AllocationFileLoaderService allocLoader = new AllocationFileLoaderService(); allocLoader.init(conf); @@ -810,19 +791,14 @@ public void testReservableQueue() throws Exception { Configuration conf = new Configuration(); conf.set(FairSchedulerConfiguration.ALLOCATION_FILE, ALLOC_FILE); - PrintWriter out = new PrintWriter(new FileWriter(ALLOC_FILE)); - out.println(""); - out.println(""); - out.println(""); - out.println(""); - out.println(""); - out.println(""); - out.println(""); - out.println(""); - out.println("DummyAgentName"); - out.println("AnyAdmissionPolicy"); - out.println(""); - out.close(); + AllocationFileWriter.create() + .addQueue(new AllocationFileQueue.Builder("reservable") + .reservation() + .build()) + .addQueue(new AllocationFileQueue.Builder("other").build()) + .reservationAgent("DummyAgentName") + .reservationPolicy("AnyAdmissionPolicy") + .writeToFile(ALLOC_FILE); AllocationFileLoaderService allocLoader = new AllocationFileLoaderService(); allocLoader.init(conf); @@ -874,15 +850,12 @@ public void testReservableCannotBeCombinedWithDynamicUserQueue() Configuration conf = new Configuration(); conf.set(FairSchedulerConfiguration.ALLOCATION_FILE, ALLOC_FILE); - PrintWriter out = new PrintWriter(new FileWriter(ALLOC_FILE)); - out.println(""); - out.println(""); - out.println(""); - out.println(""); - out.println(""); - out.println(""); - out.println(""); - out.close(); + AllocationFileWriter.create() + .addQueue(new AllocationFileQueue.Builder("notboth") + .parent(true) + .reservation() + .build()) + .writeToFile(ALLOC_FILE); AllocationFileLoaderService allocLoader = new AllocationFileLoaderService(); allocLoader.init(conf); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestAppRunnability.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestAppRunnability.java index f5819357ba0..24350838e40 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestAppRunnability.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestAppRunnability.java @@ -42,6 +42,12 @@ import org.apache.hadoop.yarn.server.resourcemanager.scheduler.event.AppAttemptRemovedSchedulerEvent; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.event.NodeAddedSchedulerEvent; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.event.NodeUpdateSchedulerEvent; + + +import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair + .allocationfile.AllocationFileQueue; +import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair + .allocationfile.AllocationFileWriter; import org.apache.hadoop.yarn.util.resource.Resources; import org.junit.After; import org.junit.Before; @@ -161,17 +167,14 @@ public void testPreemptionVariablesForQueueCreatedRuntime() throws Exception { } @Test - public void testDontAllowUndeclaredPools() throws Exception { + public void testDontAllowUndeclaredPools() { conf.setBoolean(FairSchedulerConfiguration.ALLOW_UNDECLARED_POOLS, false); conf.set(FairSchedulerConfiguration.ALLOCATION_FILE, ALLOC_FILE); - PrintWriter out = new PrintWriter(new FileWriter(ALLOC_FILE)); - out.println(""); - out.println(""); - out.println(""); - out.println(""); - out.println(""); - out.close(); + AllocationFileWriter.create() + .addQueue(new AllocationFileQueue.Builder("jerry").build()) + .writeToFile(ALLOC_FILE); + // Restarting resource manager since the file location and content is // changed. resourceManager.stop(); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestApplicationMasterServiceWithFS.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestApplicationMasterServiceWithFS.java index 2416d8cf9e5..30a0ec8dc15 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestApplicationMasterServiceWithFS.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestApplicationMasterServiceWithFS.java @@ -36,6 +36,12 @@ import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMApp; import org.apache.hadoop.yarn.server.resourcemanager.rmapp.attempt.RMAppAttempt; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.ResourceScheduler; + + +import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair + .allocationfile.AllocationFileQueue; +import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair + .allocationfile.AllocationFileWriter; import org.junit.AfterClass; import org.junit.Assert; import org.junit.BeforeClass; @@ -56,7 +62,7 @@ private static YarnConfiguration configuration; @BeforeClass - public static void setup() throws IOException { + public static void setup() { String allocFile = GenericTestUtils.getTestDir(TEST_FOLDER).getAbsolutePath(); @@ -65,21 +71,15 @@ public static void setup() throws IOException { ResourceScheduler.class); configuration.set(FairSchedulerConfiguration.ALLOCATION_FILE, allocFile); - PrintWriter out = new PrintWriter(new FileWriter(allocFile)); - out.println(""); - out.println(""); - out.println(" "); - out.println( - " 2048 mb 1 vcores"); - out.println(" "); - out.println(" "); - out.println( - " 3072 mb 1 vcores"); - out.println(" "); - out.println(" "); - out.println(" "); - out.println(""); - out.close(); + AllocationFileWriter.create() + .addQueue(new AllocationFileQueue.Builder("queueA") + .maxContainerAllocation("2048 mb 1 vcores") + .build()) + .addQueue(new AllocationFileQueue.Builder("queueB") + .maxContainerAllocation("3072 mb 1 vcores") + .build()) + .addQueue(new AllocationFileQueue.Builder("queueC").build()) + .writeToFile(allocFile); } @AfterClass diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFSAppStarvation.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFSAppStarvation.java index 9665f9a7bb0..7304d68fbeb 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFSAppStarvation.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFSAppStarvation.java @@ -22,6 +22,11 @@ import org.apache.hadoop.yarn.server.resourcemanager.MockRM; import org.apache.hadoop.yarn.server.resourcemanager.rmnode.RMNode; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.event.NodeUpdateSchedulerEvent; + +import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair + .allocationfile.AllocationFileQueue; +import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair + .allocationfile.AllocationFileWriter; import org.apache.hadoop.yarn.util.ControlledClock; import org.junit.After; @@ -187,52 +192,40 @@ private void setupClusterAndSubmitJobs() throws Exception { * 4. Submit an app that uses up all resources on the cluster */ private void setupStarvedCluster() throws IOException { - PrintWriter out = new PrintWriter(new FileWriter(ALLOC_FILE)); - out.println(""); - out.println(""); - - // Default queue - out.println(""); - out.println(""); - - // Queue with preemption disabled - out.println(""); - out.println("0" + - ""); - out.println(""); - - // Queue with minshare preemption enabled - out.println(""); - out.println("0" + - ""); - out.println("0" + - ""); - out.println("2048mb,2vcores"); - out.println(""); - - // FAIR queue with fairshare preemption enabled - out.println(""); - out.println("1" + - ""); - out.println("0" + - ""); - out.println("fair"); - addChildQueue(out, "fair"); - out.println(""); - - // DRF queue with fairshare preemption enabled - out.println(""); - out.println("1" + - ""); - out.println("0" + - ""); - out.println("drf"); - addChildQueue(out, "drf"); - out.println(""); - out.println("drf" + - ""); - out.println(""); - out.close(); + AllocationFileWriter.create() + .defaultQueueSchedulingPolicy("drf") + // Default queue + .addQueue(new AllocationFileQueue.Builder("default").build()) + // Queue with preemption disabled + .addQueue(new AllocationFileQueue.Builder("no-preemption") + .fairSharePreemptionThreshold(0).build()) + // Queue with minshare preemption enabled + .addQueue(new AllocationFileQueue.Builder("minshare") + .fairSharePreemptionThreshold(0) + .minSharePreemptionTimeout(0) + .minResources("2048mb,2vcores") + .build()) + // FAIR queue with fairshare preemption enabled + .addQueue(new AllocationFileQueue.Builder("fairshare") + .fairSharePreemptionThreshold(1) + .fairSharePreemptionTimeout(0) + .schedulingPolicy("fair") + .subQueue(new AllocationFileQueue.Builder("child") + .fairSharePreemptionThreshold(1) + .fairSharePreemptionTimeout(0) + .schedulingPolicy("fair").build()) + .build()) + // DRF queue with fairshare preemption enabled + .addQueue(new AllocationFileQueue.Builder("drf") + .fairSharePreemptionThreshold(1) + .fairSharePreemptionTimeout(0) + .schedulingPolicy("drf") + .subQueue(new AllocationFileQueue.Builder("child") + .fairSharePreemptionThreshold(1) + .fairSharePreemptionTimeout(0) + .schedulingPolicy("drf").build()) + .build()) + .writeToFile(ALLOC_FILE.getAbsolutePath()); assertTrue("Allocation file does not exist, not running the test", ALLOC_FILE.exists()); @@ -258,17 +251,6 @@ private void setupStarvedCluster() throws IOException { assertEquals(8, scheduler.getSchedulerApp(app).getLiveContainers().size()); } - private void addChildQueue(PrintWriter out, String policy) { - // Child queue under fairshare with same settings - out.println(""); - out.println("1" + - ""); - out.println("0" + - ""); - out.println("" + policy + ""); - out.println(""); - } - private void submitAppsToEachLeafQueue() { for (String queue : QUEUES) { createSchedulingRequest(1024, 1, "root." + queue, "user", 1); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFSLeafQueue.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFSLeafQueue.java index 4a738ca07fb..797a5a50335 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFSLeafQueue.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFSLeafQueue.java @@ -18,22 +18,6 @@ package org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair; -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertTrue; -import static org.mockito.Mockito.mock; - -import java.io.File; -import java.io.FileWriter; -import java.io.IOException; -import java.io.PrintWriter; -import java.util.ArrayList; -import java.util.Collection; -import java.util.Collections; -import java.util.List; -import java.util.concurrent.CountDownLatch; -import java.util.concurrent.ExecutorService; -import java.util.concurrent.TimeUnit; - import org.apache.hadoop.util.concurrent.HadoopExecutors; import org.apache.hadoop.yarn.api.records.ApplicationAttemptId; import org.apache.hadoop.yarn.api.records.Resource; @@ -45,12 +29,28 @@ import org.apache.hadoop.yarn.server.resourcemanager.scheduler.ResourceScheduler; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.event.NodeAddedSchedulerEvent; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.event.NodeUpdateSchedulerEvent; +import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.allocationfile.AllocationFileQueue; +import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.allocationfile.AllocationFileWriter; import org.apache.hadoop.yarn.util.resource.Resources; import org.junit.After; import org.junit.Before; import org.junit.Test; import org.mockito.Mockito; +import java.io.File; +import java.io.IOException; +import java.util.ArrayList; +import java.util.Collection; +import java.util.Collections; +import java.util.List; +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.TimeUnit; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertTrue; +import static org.mockito.Mockito.mock; + public class TestFSLeafQueue extends FairSchedulerTestBase { private final static String ALLOC_FILE = new File(TEST_DIR, TestFSLeafQueue.class.getName() + ".xml").getAbsolutePath(); @@ -100,15 +100,13 @@ public void testUpdateDemand() { } @Test (timeout = 5000) - public void test() throws Exception { + public void test() { conf.set(FairSchedulerConfiguration.ALLOCATION_FILE, ALLOC_FILE); - PrintWriter out = new PrintWriter(new FileWriter(ALLOC_FILE)); - out.println(""); - out.println(""); - out.println(""); - out.println(""); - out.println(""); - out.close(); + + AllocationFileWriter.create() + .addQueue(new AllocationFileQueue.Builder("queueA").build()) + .addQueue(new AllocationFileQueue.Builder("queueB").build()) + .writeToFile(ALLOC_FILE); resourceManager = new MockRM(conf); resourceManager.start(); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFairScheduler.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFairScheduler.java index 0d6caebac68..771bb738f5b 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFairScheduler.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFairScheduler.java @@ -116,6 +116,17 @@ import org.apache.hadoop.yarn.server.resourcemanager.scheduler.event.NodeRemovedSchedulerEvent; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.event.NodeUpdateSchedulerEvent; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.QueuePlacementRule.Default; + +import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair + .allocationfile.AllocationFileQueue; +import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair + .allocationfile.AllocationFileQueuePlacementPolicy; +import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair + .allocationfile.AllocationFileQueuePlacementRule; +import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair + .allocationfile.AllocationFileWriter; +import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair + .allocationfile.UserSettings; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.policies.DominantResourceFairnessPolicy; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.policies.FifoPolicy; import org.apache.hadoop.yarn.server.utils.BuilderUtils; @@ -343,27 +354,18 @@ public void testQueueMaximumCapacityAllocations() throws IOException { int tooHighQueueAllocation = RM_SCHEDULER_MAXIMUM_ALLOCATION_MB_VALUE +1; - PrintWriter out = new PrintWriter(new FileWriter(ALLOC_FILE)); - out.println(""); - out.println(""); - out.println(" "); - out.println( - " 512 mb 1 vcores"); - out.println(" "); - out.println(" "); - out.println(" "); - out.println(" "); - out.println( - " 2048 mb 3 vcores"); - out.println(" "); - out.println(" "); - out.println(" "); - out.println(" "); - out.println(" " + tooHighQueueAllocation - + " mb 1 vcores"); - out.println(" "); - out.println(""); - out.close(); + AllocationFileWriter.create() + .addQueue(new AllocationFileQueue.Builder("queueA") + .maxContainerAllocation("512 mb 1 vcores").build()) + .addQueue(new AllocationFileQueue.Builder("queueB").build()) + .addQueue(new AllocationFileQueue.Builder("queueC") + .maxContainerAllocation("2048 mb 3 vcores") + .subQueue(new AllocationFileQueue.Builder("queueD").build()) + .build()) + .addQueue(new AllocationFileQueue.Builder("queueE") + .maxContainerAllocation(tooHighQueueAllocation + " mb 1 vcores") + .build()) + .writeToFile(ALLOC_FILE); scheduler.init(conf); @@ -399,17 +401,12 @@ public void testNormalizationUsingQueueMaximumAllocation() int queueMaxAllocation = 4096; conf.set(FairSchedulerConfiguration.ALLOCATION_FILE, ALLOC_FILE); - PrintWriter out = new PrintWriter(new FileWriter(ALLOC_FILE)); - out.println(""); - out.println(""); - out.println(" "); - out.println(" " + queueMaxAllocation - + " mb 1 vcores" + ""); - out.println(" "); - out.println(" "); - out.println(" "); - out.println(""); - out.close(); + AllocationFileWriter.create() + .addQueue(new AllocationFileQueue.Builder("queueA") + .maxContainerAllocation(queueMaxAllocation + " mb 1 vcores") + .build()) + .addQueue(new AllocationFileQueue.Builder("queueB").build()) + .writeToFile(ALLOC_FILE); scheduler.init(conf); scheduler.start(); @@ -454,19 +451,17 @@ public void testFairShareWithHighMaxResources() throws IOException { // set queueA and queueB maxResources, // the sum of queueA and queueB maxResources is more than // Integer.MAX_VALUE. - PrintWriter out = new PrintWriter(new FileWriter(ALLOC_FILE)); - out.println(""); - out.println(""); - out.println(""); - out.println("1073741824 mb 1000 vcores"); - out.println(".25"); - out.println(""); - out.println(""); - out.println("1073741824 mb 1000 vcores"); - out.println(".75"); - out.println(""); - out.println(""); - out.close(); + + AllocationFileWriter.create() + .addQueue(new AllocationFileQueue.Builder("queueA") + .maxResources("1073741824 mb 1000 vcores") + .weight(.25f) + .build()) + .addQueue(new AllocationFileQueue.Builder("queueB") + .maxResources("1073741824 mb 1000 vcores") + .weight(.75f) + .build()) + .writeToFile(ALLOC_FILE); scheduler.init(conf); scheduler.start(); @@ -508,18 +503,16 @@ public void testFairShareWithHighMaxResources() throws IOException { public void testFairShareWithLowMaxResources() throws IOException { PrintWriter out = new PrintWriter(new FileWriter(ALLOC_FILE)); - out.println(""); - out.println(""); - out.println(" "); - out.println(" 1024 mb 1 vcores"); - out.println(" 0.75"); - out.println(" "); - out.println(" "); - out.println(" 3072 mb 3 vcores"); - out.println(" 0.25"); - out.println(" "); - out.println(""); - out.close(); + AllocationFileWriter.create() + .addQueue(new AllocationFileQueue.Builder("queueA") + .maxResources("1024 mb 1 vcores") + .weight(.75f) + .build()) + .addQueue(new AllocationFileQueue.Builder("queueB") + .maxResources("3072 mb 3 vcores") + .weight(.25f) + .build()) + .writeToFile(ALLOC_FILE); conf.set(FairSchedulerConfiguration.ALLOCATION_FILE, ALLOC_FILE); scheduler.init(conf); @@ -578,15 +571,12 @@ public void testFairShareWithLowMaxResources() throws IOException { */ @Test public void testChildMaxResources() throws IOException { - PrintWriter out = new PrintWriter(new FileWriter(ALLOC_FILE)); - - out.println(""); - out.println(""); - out.println(" "); - out.println(" 2048mb,2vcores"); - out.println(" "); - out.println(""); - out.close(); + AllocationFileWriter.create() + .addQueue(new AllocationFileQueue.Builder("queueA") + .parent(true) + .maxChildResources("2048mb,2vcores") + .build()) + .writeToFile(ALLOC_FILE); conf.set(FairSchedulerConfiguration.ALLOCATION_FILE, ALLOC_FILE); scheduler.init(conf); @@ -625,14 +615,12 @@ public void testChildMaxResources() throws IOException { assertEquals("App 2 is not running with the correct number of containers", 2, scheduler.getSchedulerApp(attId2).getLiveContainers().size()); - out = new PrintWriter(new FileWriter(ALLOC_FILE)); - out.println(""); - out.println(""); - out.println(" "); - out.println(" 3072mb,3vcores"); - out.println(" "); - out.println(""); - out.close(); + AllocationFileWriter.create() + .addQueue(new AllocationFileQueue.Builder("queueA") + .parent(true) + .maxChildResources("3072mb,3vcores") + .build()) + .writeToFile(ALLOC_FILE); scheduler.reinitialize(conf, resourceManager.getRMContext()); @@ -648,14 +636,12 @@ public void testChildMaxResources() throws IOException { assertEquals("App 2 is not running with the correct number of containers", 3, scheduler.getSchedulerApp(attId2).getLiveContainers().size()); - out = new PrintWriter(new FileWriter(ALLOC_FILE)); - out.println(""); - out.println(""); - out.println(" "); - out.println(" 1024mb,1vcores"); - out.println(" "); - out.println(""); - out.close(); + AllocationFileWriter.create() + .addQueue(new AllocationFileQueue.Builder("queueA") + .parent(true) + .maxChildResources("1024mb,1vcores") + .build()) + .writeToFile(ALLOC_FILE); scheduler.reinitialize(conf, resourceManager.getRMContext()); @@ -673,17 +659,13 @@ public void testChildMaxResources() throws IOException { public void testFairShareWithZeroWeight() throws IOException { conf.set(FairSchedulerConfiguration.ALLOCATION_FILE, ALLOC_FILE); // set queueA and queueB weight zero. - PrintWriter out = new PrintWriter(new FileWriter(ALLOC_FILE)); - out.println(""); - out.println(""); - out.println(""); - out.println("0.0"); - out.println(""); - out.println(""); - out.println("0.0"); - out.println(""); - out.println(""); - out.close(); + + AllocationFileWriter.create() + .addQueue(new AllocationFileQueue.Builder("queueA") + .weight(0.0f).build()) + .addQueue(new AllocationFileQueue.Builder("queueB") + .weight(0.0f).build()) + .writeToFile(ALLOC_FILE); scheduler.init(conf); scheduler.start(); @@ -721,26 +703,23 @@ public void testFairShareWithZeroWeight() throws IOException { @Test public void testComputeMaxAMResource() throws IOException { conf.set(FairSchedulerConfiguration.ALLOCATION_FILE, ALLOC_FILE); - PrintWriter out = new PrintWriter(new FileWriter(ALLOC_FILE)); - out.println(""); - out.println(""); - out.println(""); - out.println("0"); - out.println("0.5"); - out.println("4096 mb 4 vcores"); - out.println(""); - out.println(""); - out.println("0.0"); - out.println("0.5"); - out.println(""); - out.println(""); - out.println("1"); - out.println("0.5"); - out.println(""); - out.println("drf" + - ""); - out.println(""); - out.close(); + + AllocationFileWriter.create() + .addQueue(new AllocationFileQueue.Builder("queueFSZeroWithMax") + .weight(0) + .maxAMShare(0.5) + .maxResources("4096 mb 4 vcores") + .build()) + .addQueue(new AllocationFileQueue.Builder("queueFSZeroWithAVL") + .weight(0.0f) + .maxAMShare(0.5) + .build()) + .addQueue(new AllocationFileQueue.Builder("queueFSNonZero") + .weight(1) + .maxAMShare(0.5) + .build()) + .defaultQueueSchedulingPolicy("drf") + .writeToFile(ALLOC_FILE); scheduler.init(conf); scheduler.start(); @@ -855,19 +834,16 @@ public void testFairShareWithZeroWeightNoneZeroMinRes() throws IOException { conf.set(FairSchedulerConfiguration.ALLOCATION_FILE, ALLOC_FILE); // set queueA and queueB weight zero. // set queueA and queueB minResources 1. - PrintWriter out = new PrintWriter(new FileWriter(ALLOC_FILE)); - out.println(""); - out.println(""); - out.println(""); - out.println("1 mb 1 vcores"); - out.println("0.0"); - out.println(""); - out.println(""); - out.println("1 mb 1 vcores"); - out.println("0.0"); - out.println(""); - out.println(""); - out.close(); + AllocationFileWriter.create() + .addQueue(new AllocationFileQueue.Builder("queueA") + .weight(0) + .minResources("1 mb 1 vcores") + .build()) + .addQueue(new AllocationFileQueue.Builder("queueB") + .minResources("1 mb 1 vcores") + .weight(0.0f) + .build()) + .writeToFile(ALLOC_FILE); scheduler.init(conf); scheduler.start(); @@ -905,19 +881,16 @@ public void testFairShareWithNoneZeroWeightNoneZeroMinRes() conf.set(FairSchedulerConfiguration.ALLOCATION_FILE, ALLOC_FILE); // set queueA and queueB weight 0.5. // set queueA and queueB minResources 1024. - PrintWriter out = new PrintWriter(new FileWriter(ALLOC_FILE)); - out.println(""); - out.println(""); - out.println(""); - out.println("1024 mb 1 vcores"); - out.println("0.5"); - out.println(""); - out.println(""); - out.println("1024 mb 1 vcores"); - out.println("0.5"); - out.println(""); - out.println(""); - out.close(); + AllocationFileWriter.create() + .addQueue(new AllocationFileQueue.Builder("queueA") + .weight(0.5f) + .minResources("1024 mb 1 vcores") + .build()) + .addQueue(new AllocationFileQueue.Builder("queueB") + .weight(0.5f) + .minResources("1024 mb 1 vcores") + .build()) + .writeToFile(ALLOC_FILE); scheduler.init(conf); scheduler.start(); @@ -953,17 +926,14 @@ public void testFairShareWithNoneZeroWeightNoneZeroMinRes() public void testQueueInfo() throws IOException { conf.set(FairSchedulerConfiguration.ALLOCATION_FILE, ALLOC_FILE); - PrintWriter out = new PrintWriter(new FileWriter(ALLOC_FILE)); - out.println(""); - out.println(""); - out.println(""); - out.println(".25"); - out.println(""); - out.println(""); - out.println(".75"); - out.println(""); - out.println(""); - out.close(); + AllocationFileWriter.create() + .addQueue(new AllocationFileQueue.Builder("queueA") + .weight(0.25f) + .build()) + .addQueue(new AllocationFileQueue.Builder("queueB") + .weight(0.75f) + .build()) + .writeToFile(ALLOC_FILE); scheduler.init(conf); scheduler.start(); @@ -1477,19 +1447,15 @@ public void testReservationThresholdWithAssignMultiple() throws Exception { public void testContainerReservationAttemptExceedingQueueMax() throws Exception { conf.set(FairSchedulerConfiguration.ALLOCATION_FILE, ALLOC_FILE); - PrintWriter out = new PrintWriter(new FileWriter(ALLOC_FILE)); - out.println(""); - out.println(""); - out.println(""); - out.println(""); - out.println("2048mb,5vcores"); - out.println(""); - out.println(""); - out.println("2048mb,10vcores"); - out.println(""); - out.println(""); - out.println(""); - out.close(); + + AllocationFileWriter.create() + .addQueue(new AllocationFileQueue.Builder("root") + .subQueue(new AllocationFileQueue.Builder("queue1") + .maxResources("2048mb,5vcores").build()) + .subQueue(new AllocationFileQueue.Builder("queue2") + .maxResources("2048mb,10vcores").build()) + .build()) + .writeToFile(ALLOC_FILE); scheduler.init(conf); scheduler.start(); @@ -1539,22 +1505,20 @@ public void testContainerReservationAttemptExceedingQueueMax() public void testRequestAMResourceInZeroFairShareQueue() throws Exception { conf.set(FairSchedulerConfiguration.ALLOCATION_FILE, ALLOC_FILE); - PrintWriter out = new PrintWriter(new FileWriter(ALLOC_FILE)); - out.println(""); - out.println(""); - out.println(""); - out.println("0.0"); - out.println("4096mb,10vcores"); - out.println("0.5"); - out.println(""); - out.println(""); - out.println("2.0"); - out.println(""); - out.println(""); - out.println("0.000001"); - out.println(""); - out.println(""); - out.close(); + AllocationFileWriter.create() + .addQueue(new AllocationFileQueue.Builder("queue1") + .weight(0) + .maxAMShare(0.5) + .maxResources("4096mb,10vcores") + .build()) + .addQueue(new AllocationFileQueue.Builder("queue2") + .weight(2.0f) + .build()) + .addQueue(new AllocationFileQueue.Builder("queue3") + .weight(0.000001f) + .build()) + .defaultQueueSchedulingPolicy("drf") + .writeToFile(ALLOC_FILE); scheduler.init(conf); scheduler.start(); @@ -1612,19 +1576,15 @@ public void testRequestAMResourceInZeroFairShareQueue() throws Exception { @Test (timeout = 500000) public void testContainerReservationNotExceedingQueueMax() throws Exception { conf.set(FairSchedulerConfiguration.ALLOCATION_FILE, ALLOC_FILE); - PrintWriter out = new PrintWriter(new FileWriter(ALLOC_FILE)); - out.println(""); - out.println(""); - out.println(""); - out.println(""); - out.println("3072mb,10vcores"); - out.println(""); - out.println(""); - out.println("2048mb,10vcores"); - out.println(""); - out.println(""); - out.println(""); - out.close(); + + AllocationFileWriter.create() + .addQueue(new AllocationFileQueue.Builder("root") + .subQueue(new AllocationFileQueue.Builder("queue1") + .maxResources("3072mb,10vcores").build()) + .subQueue(new AllocationFileQueue.Builder("queue2") + .maxResources("2048mb,10vcores").build()) + .build()) + .writeToFile(ALLOC_FILE); scheduler.init(conf); scheduler.start(); @@ -1673,19 +1633,14 @@ public void testContainerReservationNotExceedingQueueMax() throws Exception { getCurrentReservation().getMemorySize()); // Now reduce max Resources of queue1 down to 2048 - out = new PrintWriter(new FileWriter(ALLOC_FILE)); - out.println(""); - out.println(""); - out.println(""); - out.println(""); - out.println("2048mb,10vcores"); - out.println(""); - out.println(""); - out.println("2048mb,10vcores"); - out.println(""); - out.println(""); - out.println(""); - out.close(); + AllocationFileWriter.create() + .addQueue(new AllocationFileQueue.Builder("root") + .subQueue(new AllocationFileQueue.Builder("queue1") + .maxResources("2048mb,10vcores").build()) + .subQueue(new AllocationFileQueue.Builder("queue2") + .maxResources("2048mb,10vcores").build()) + .build()) + .writeToFile(ALLOC_FILE); scheduler.reinitialize(conf, resourceManager.getRMContext()); @@ -1706,13 +1661,9 @@ public void testContainerReservationNotExceedingQueueMax() throws Exception { public void testReservationThresholdGatesReservations() throws Exception { conf.set(FairSchedulerConfiguration.ALLOCATION_FILE, ALLOC_FILE); - PrintWriter out = new PrintWriter(new FileWriter(ALLOC_FILE)); - out.println(""); - out.println(""); - out.println("drf" + - ""); - out.println(""); - out.close(); + AllocationFileWriter.create() + .defaultQueueSchedulingPolicy("drf") + .writeToFile(ALLOC_FILE); // Set threshold to 2 * 1024 ==> 2048 MB & 2 * 1 ==> 2 vcores (test will // use vcores) @@ -1870,15 +1821,14 @@ public void testAssignToQueue() throws Exception { public void testAssignToBadDefaultQueue() throws Exception { conf.set(FairSchedulerConfiguration.ALLOCATION_FILE, ALLOC_FILE); - PrintWriter out = new PrintWriter(new FileWriter(ALLOC_FILE)); - out.println(""); - out.println(""); - out.println(""); - out.println(""); - out.println(""); - out.println(""); - out.println(""); - out.close(); + AllocationFileWriter.create() + .queuePlacementPolicy(new AllocationFileQueuePlacementPolicy() + .addRule(new AllocationFileQueuePlacementRule("specified") + .create(false)) + .addRule(new AllocationFileQueuePlacementRule("default") + .create(false))) + .writeToFile(ALLOC_FILE); + scheduler.init(conf); scheduler.start(); scheduler.reinitialize(conf, resourceManager.getRMContext()); @@ -1967,17 +1917,13 @@ public void testQueuePlacementWithPolicy() throws Exception { public void testFairShareWithMinAlloc() throws Exception { conf.set(FairSchedulerConfiguration.ALLOCATION_FILE, ALLOC_FILE); - PrintWriter out = new PrintWriter(new FileWriter(ALLOC_FILE)); - out.println(""); - out.println(""); - out.println(""); - out.println("1024mb,0vcores"); - out.println(""); - out.println(""); - out.println("2048mb,0vcores"); - out.println(""); - out.println(""); - out.close(); + AllocationFileWriter.create() + .addQueue(new AllocationFileQueue.Builder("queueA") + .minResources("1024mb,0vcores").build()) + .addQueue(new AllocationFileQueue.Builder("queueB") + .minResources("2048mb,0vcores") + .build()) + .writeToFile(ALLOC_FILE); scheduler.init(conf); scheduler.start(); @@ -2013,21 +1959,22 @@ public void testNestedUserQueue() throws IOException { conf.set(FairSchedulerConfiguration.ALLOCATION_FILE, ALLOC_FILE); conf.setClass(CommonConfigurationKeys.HADOOP_SECURITY_GROUP_MAPPING, SimpleGroupsMapping.class, GroupMappingServiceProvider.class); - PrintWriter out = new PrintWriter(new FileWriter(ALLOC_FILE)); - out.println(""); - out.println(""); - out.println(""); - out.println("1024mb,0vcores"); - out.println(""); - out.println(""); - out.println(""); - out.println(""); - out.println(" "); - out.println(""); - out.println(""); - out.println(""); - out.println(""); - out.close(); + + AllocationFileWriter.create() + .addQueue(new AllocationFileQueue.Builder("user1group") + .parent(true) + .minResources("1024mb,0vcores") + .build()) + .queuePlacementPolicy(new AllocationFileQueuePlacementPolicy() + .addRule(new AllocationFileQueuePlacementRule("specified") + .create(false)) + .addRule(new AllocationFileQueuePlacementRule("nestedUserQueue") + .addNestedRule( + new AllocationFileQueuePlacementRule("primaryGroup") + .create(false))) + .addRule(new AllocationFileQueuePlacementRule("default") + .create(false))) + .writeToFile(ALLOC_FILE); scheduler.init(conf); scheduler.start(); @@ -2044,21 +1991,18 @@ public void testNestedUserQueue() throws IOException { public void testFairShareAndWeightsInNestedUserQueueRule() throws Exception { conf.set(FairSchedulerConfiguration.ALLOCATION_FILE, ALLOC_FILE); - PrintWriter out = new PrintWriter(new FileWriter(ALLOC_FILE)); - - out.println(""); - out.println(""); - out.println(""); - out.println("1024mb,0vcores"); - out.println(""); - out.println(""); - out.println(""); - out.println(" "); - out.println(""); - out.println(""); - out.println(""); - out.println(""); - out.close(); + AllocationFileWriter.create() + .addQueue(new AllocationFileQueue.Builder("parentq") + .parent(true) + .minResources("1024mb,0vcores") + .build()) + .queuePlacementPolicy(new AllocationFileQueuePlacementPolicy() + .addRule(new AllocationFileQueuePlacementRule("nestedUserQueue") + .addNestedRule( + new AllocationFileQueuePlacementRule("specified") + .create(false))) + .addRule(new AllocationFileQueuePlacementRule("default"))) + .writeToFile(ALLOC_FILE); RMApp rmApp1 = new MockRMApp(0, 0, RMAppState.NEW); RMApp rmApp2 = new MockRMApp(1, 1, RMAppState.NEW); @@ -2105,21 +2049,16 @@ public void testFairShareAndWeightsInNestedUserQueueRule() throws Exception { public void testSteadyFairShareWithReloadAndNodeAddRemove() throws Exception { conf.set(FairSchedulerConfiguration.ALLOCATION_FILE, ALLOC_FILE); - PrintWriter out = new PrintWriter(new FileWriter(ALLOC_FILE)); - out.println(""); - out.println(""); - out.println("fair"); - out.println(""); - out.println(" drf"); - out.println(" "); - out.println(" 1"); - out.println(" "); - out.println(" "); - out.println(" 1"); - out.println(" "); - out.println(""); - out.println(""); - out.close(); + AllocationFileWriter.create() + .defaultQueueSchedulingPolicy("fair") + .addQueue(new AllocationFileQueue.Builder("root") + .schedulingPolicy("drf") + .subQueue(new AllocationFileQueue.Builder("child1") + .weight(1).build()) + .subQueue(new AllocationFileQueue.Builder("child2") + .weight(1).build()) + .build()) + .writeToFile(ALLOC_FILE); scheduler.init(conf); scheduler.start(); @@ -2147,24 +2086,19 @@ public void testSteadyFairShareWithReloadAndNodeAddRemove() throws Exception { .getSteadyFairShare().getMemorySize()); // Reload the allocation configuration file - out = new PrintWriter(new FileWriter(ALLOC_FILE)); - out.println(""); - out.println(""); - out.println("fair"); - out.println(""); - out.println(" drf"); - out.println(" "); - out.println(" 1"); - out.println(" "); - out.println(" "); - out.println(" 2"); - out.println(" "); - out.println(" "); - out.println(" 2"); - out.println(" "); - out.println(""); - out.println(""); - out.close(); + AllocationFileWriter.create() + .defaultQueueSchedulingPolicy("fair") + .addQueue(new AllocationFileQueue.Builder("root") + .schedulingPolicy("drf") + .subQueue(new AllocationFileQueue.Builder("child1") + .weight(1).build()) + .subQueue(new AllocationFileQueue.Builder("child2") + .weight(2).build()) + .subQueue(new AllocationFileQueue.Builder("child3") + .weight(2).build()) + .build()) + .writeToFile(ALLOC_FILE); + scheduler.reinitialize(conf, resourceManager.getRMContext()); // The steady fair shares for all queues should be updated @@ -2281,27 +2215,19 @@ public void testQueueDemandCalculation() throws Exception { } @Test - public void testHierarchicalQueueAllocationFileParsing() throws IOException, SAXException, - AllocationConfigurationException, ParserConfigurationException { + public void testHierarchicalQueueAllocationFileParsing() throws IOException { conf.set(FairSchedulerConfiguration.ALLOCATION_FILE, ALLOC_FILE); - - PrintWriter out = new PrintWriter(new FileWriter(ALLOC_FILE)); - out.println(""); - out.println(""); - out.println(""); - out.println("2048mb,0vcores"); - out.println(""); - out.println(""); - out.println("2048mb,0vcores"); - out.println(""); - out.println("2048mb,0vcores"); - out.println(""); - out.println(""); - out.println("2048mb,0vcores"); - out.println(""); - out.println(""); - out.println(""); - out.close(); + AllocationFileWriter.create() + .addQueue(new AllocationFileQueue.Builder("queueA") + .minResources("2048mb,0vcores").build()) + .addQueue(new AllocationFileQueue.Builder("queueB") + .minResources("2048mb,0vcores") + .subQueue(new AllocationFileQueue.Builder("queueC") + .minResources("2048mb,0vcores").build()) + .subQueue(new AllocationFileQueue.Builder("queueD") + .minResources("2048mb,0vcores").build()) + .build()) + .writeToFile(ALLOC_FILE); scheduler.init(conf); scheduler.start(); @@ -2322,27 +2248,22 @@ public void testHierarchicalQueueAllocationFileParsing() throws IOException, SAX public void testConfigureRootQueue() throws Exception { conf.set(FairSchedulerConfiguration.ALLOCATION_FILE, ALLOC_FILE); - PrintWriter out = new PrintWriter(new FileWriter(ALLOC_FILE)); - out.println(""); - out.println(""); - out.println("fair"); - out.println(""); - out.println(" drf"); - out.println(" "); - out.println(" 1024mb,1vcores"); - out.println(" "); - out.println(" "); - out.println(" 1024mb,4vcores"); - out.println(" "); - out.println(" 100"); - out.println(" 120"); - out.println(" .5"); - out.println(""); - out.println("300"); - out.println("200"); - out.println(".6"); - out.println(""); - out.close(); + AllocationFileWriter.create() + .defaultQueueSchedulingPolicy("fair") + .defaultFairSharePreemptionTimeout(300) + .defaultMinSharePreemptionTimeout(200) + .defaultFairSharePreemptionThreshold(.6) + .addQueue(new AllocationFileQueue.Builder("root") + .schedulingPolicy("drf") + .fairSharePreemptionTimeout(100) + .fairSharePreemptionThreshold(.5) + .minSharePreemptionTimeout(120) + .subQueue(new AllocationFileQueue.Builder("child1") + .minResources("1024mb,1vcores").build()) + .subQueue(new AllocationFileQueue.Builder("child2") + .minResources("1024mb,4vcores").build()) + .build()) + .writeToFile(ALLOC_FILE); scheduler.init(conf); scheduler.start(); @@ -2397,14 +2318,10 @@ public void testUserMaxRunningApps() throws Exception { // Set max running apps conf.set(FairSchedulerConfiguration.ALLOCATION_FILE, ALLOC_FILE); - PrintWriter out = new PrintWriter(new FileWriter(ALLOC_FILE)); - out.println(""); - out.println(""); - out.println(""); - out.println("1"); - out.println(""); - out.println(""); - out.close(); + AllocationFileWriter.create() + .userSettings(new UserSettings.Builder("user1") + .maxRunningApps(1).build()) + .writeToFile(ALLOC_FILE); scheduler.init(conf); scheduler.start(); @@ -2449,65 +2366,57 @@ public void testUserMaxRunningApps() throws Exception { @Test (timeout = 5000) public void testIncreaseQueueMaxRunningAppsOnTheFly() throws Exception { - String allocBefore = "" + - "" + - "" + - "" + - "1" + - "" + - "" + - ""; - - String allocAfter = "" + - "" + - "" + - "" + - "3" + - "" + - "" + - ""; + AllocationFileWriter allocBefore = AllocationFileWriter.create() + .addQueue(new AllocationFileQueue.Builder("root") + .subQueue( + new AllocationFileQueue.Builder("queue1") + .maxRunningApps(1) + .build()) + .build()); + + AllocationFileWriter allocAfter = AllocationFileWriter.create() + .addQueue(new AllocationFileQueue.Builder("root") + .subQueue( + new AllocationFileQueue.Builder("queue1") + .maxRunningApps(3) + .build()) + .build()); testIncreaseQueueSettingOnTheFlyInternal(allocBefore, allocAfter); } @Test (timeout = 5000) public void testIncreaseUserMaxRunningAppsOnTheFly() throws Exception { - String allocBefore = ""+ - ""+ - ""+ - ""+ - "10"+ - ""+ - ""+ - ""+ - "1"+ - ""+ - ""; - - String allocAfter = ""+ - ""+ - ""+ - ""+ - "10"+ - ""+ - ""+ - ""+ - "3"+ - ""+ - ""; + AllocationFileWriter allocBefore = AllocationFileWriter.create() + .addQueue(new AllocationFileQueue.Builder("root") + .subQueue( + new AllocationFileQueue.Builder("queue1") + .maxRunningApps(10) + .build()) + .build()) + .userSettings(new UserSettings.Builder("user1") + .maxRunningApps(1).build()); + + AllocationFileWriter allocAfter = AllocationFileWriter.create() + .addQueue(new AllocationFileQueue.Builder("root") + .subQueue( + new AllocationFileQueue.Builder("queue1") + .maxRunningApps(10) + .build()) + .build()) + .userSettings(new UserSettings.Builder("user1") + .maxRunningApps(3).build()); testIncreaseQueueSettingOnTheFlyInternal(allocBefore, allocAfter); } - private void testIncreaseQueueSettingOnTheFlyInternal(String allocBefore, - String allocAfter) throws Exception { + private void testIncreaseQueueSettingOnTheFlyInternal( + AllocationFileWriter allocBefore, + AllocationFileWriter allocAfter) throws Exception { // Set max running apps conf.set(FairSchedulerConfiguration.ALLOCATION_FILE, ALLOC_FILE); - PrintWriter out = new PrintWriter(new FileWriter(ALLOC_FILE)); - out.println(allocBefore); - out.close(); - + allocBefore.writeToFile(ALLOC_FILE); scheduler.init(conf); scheduler.start(); scheduler.reinitialize(conf, resourceManager.getRMContext()); @@ -2555,9 +2464,7 @@ private void testIncreaseQueueSettingOnTheFlyInternal(String allocBefore, // App 4 should not be running assertEquals(0, scheduler.getSchedulerApp(attId4).getLiveContainers().size()); - out = new PrintWriter(new FileWriter(ALLOC_FILE)); - out.println(allocAfter); - out.close(); + allocAfter.writeToFile(ALLOC_FILE); scheduler.reinitialize(conf, resourceManager.getRMContext()); scheduler.update(); @@ -2592,64 +2499,56 @@ private void testIncreaseQueueSettingOnTheFlyInternal(String allocBefore, @Test (timeout = 5000) public void testDecreaseQueueMaxRunningAppsOnTheFly() throws Exception { - String allocBefore = "" + - "" + - "" + - "" + - "3" + - "" + - "" + - ""; - - String allocAfter = "" + - "" + - "" + - "" + - "1" + - "" + - "" + - ""; + AllocationFileWriter allocBefore = AllocationFileWriter.create() + .addQueue(new AllocationFileQueue.Builder("root") + .subQueue( + new AllocationFileQueue.Builder("queue1") + .maxRunningApps(3) + .build()) + .build()); + + AllocationFileWriter allocAfter = AllocationFileWriter.create() + .addQueue(new AllocationFileQueue.Builder("root") + .subQueue( + new AllocationFileQueue.Builder("queue1") + .maxRunningApps(1) + .build()) + .build()); testDecreaseQueueSettingOnTheFlyInternal(allocBefore, allocAfter); } @Test (timeout = 5000) public void testDecreaseUserMaxRunningAppsOnTheFly() throws Exception { - String allocBefore = ""+ - ""+ - ""+ - ""+ - "10"+ - ""+ - ""+ - ""+ - "3"+ - ""+ - ""; - - String allocAfter = ""+ - ""+ - ""+ - ""+ - "10"+ - ""+ - ""+ - ""+ - "1"+ - ""+ - ""; + AllocationFileWriter allocBefore = AllocationFileWriter.create() + .addQueue(new AllocationFileQueue.Builder("root") + .subQueue( + new AllocationFileQueue.Builder("queue1") + .maxRunningApps(10) + .build()) + .build()) + .userSettings(new UserSettings.Builder("user1") + .maxRunningApps(3).build()); + + AllocationFileWriter allocAfter = AllocationFileWriter.create() + .addQueue(new AllocationFileQueue.Builder("root") + .subQueue( + new AllocationFileQueue.Builder("queue1") + .maxRunningApps(10) + .build()) + .build()) + .userSettings(new UserSettings.Builder("user1") + .maxRunningApps(1).build()); testDecreaseQueueSettingOnTheFlyInternal(allocBefore, allocAfter); } - private void testDecreaseQueueSettingOnTheFlyInternal(String allocBefore, - String allocAfter) throws Exception { + private void testDecreaseQueueSettingOnTheFlyInternal( + AllocationFileWriter allocBefore, + AllocationFileWriter allocAfter) throws Exception { // Set max running apps conf.set(FairSchedulerConfiguration.ALLOCATION_FILE, ALLOC_FILE); - - PrintWriter out = new PrintWriter(new FileWriter(ALLOC_FILE)); - out.println(allocBefore); - out.close(); + allocBefore.writeToFile(ALLOC_FILE); scheduler.init(conf); scheduler.start(); @@ -2698,9 +2597,7 @@ private void testDecreaseQueueSettingOnTheFlyInternal(String allocBefore, // App 4 should not be running assertEquals(0, scheduler.getSchedulerApp(attId4).getLiveContainers().size()); - out = new PrintWriter(new FileWriter(ALLOC_FILE)); - out.println(allocAfter); - out.close(); + allocAfter.writeToFile(ALLOC_FILE); scheduler.reinitialize(conf, resourceManager.getRMContext()); scheduler.update(); @@ -2813,19 +2710,16 @@ public void testAclSubmitApplication() throws Exception { // Set acl's conf.set(FairSchedulerConfiguration.ALLOCATION_FILE, ALLOC_FILE); - PrintWriter out = new PrintWriter(new FileWriter(ALLOC_FILE)); - out.println(""); - out.println(""); - out.println(""); - out.println(" "); - out.println(" "); - out.println(" "); - out.println(" norealuserhasthisname"); - out.println(" norealuserhasthisname"); - out.println(" "); - out.println(""); - out.println(""); - out.close(); + AllocationFileWriter.create() + .addQueue(new AllocationFileQueue.Builder("root") + .aclSubmitApps(" ") + .aclAdministerApps(" ") + .subQueue(new AllocationFileQueue.Builder("queue1") + .aclSubmitApps("norealuserhasthisname") + .aclAdministerApps("norealuserhasthisname") + .build()) + .build()) + .writeToFile(ALLOC_FILE); scheduler.init(conf); scheduler.start(); @@ -3150,19 +3044,17 @@ public void testAssignContainer() throws Exception { public void testNotAllowSubmitApplication() throws Exception { // Set acl's conf.set(FairSchedulerConfiguration.ALLOCATION_FILE, ALLOC_FILE); - PrintWriter out = new PrintWriter(new FileWriter(ALLOC_FILE)); - out.println(""); - out.println(""); - out.println(""); - out.println(" "); - out.println(" "); - out.println(" "); - out.println(" userallow"); - out.println(" userallow"); - out.println(" "); - out.println(""); - out.println(""); - out.close(); + + AllocationFileWriter.create() + .addQueue(new AllocationFileQueue.Builder("root") + .aclSubmitApps(" ") + .aclAdministerApps(" ") + .subQueue(new AllocationFileQueue.Builder("queue1") + .aclSubmitApps("userallow") + .aclAdministerApps("userallow") + .build()) + .build()) + .writeToFile(ALLOC_FILE); scheduler.init(conf); scheduler.start(); @@ -3697,17 +3589,12 @@ private void verifyQueueNumRunnable(String queueName, int numRunnableInQueue, public void testUserAndQueueMaxRunningApps() throws Exception { conf.set(FairSchedulerConfiguration.ALLOCATION_FILE, ALLOC_FILE); - PrintWriter out = new PrintWriter(new FileWriter(ALLOC_FILE)); - out.println(""); - out.println(""); - out.println(""); - out.println("2"); - out.println(""); - out.println(""); - out.println("1"); - out.println(""); - out.println(""); - out.close(); + AllocationFileWriter.create() + .addQueue(new AllocationFileQueue.Builder("queue1") + .maxRunningApps(2).build()) + .userSettings(new UserSettings.Builder("user1") + .maxRunningApps(1).build()) + .writeToFile(ALLOC_FILE); scheduler.init(conf); scheduler.start(); @@ -3750,14 +3637,10 @@ public void testMultipleCompletedEvent() throws Exception { // Set up a fair scheduler conf.set(FairSchedulerConfiguration.ALLOCATION_FILE, ALLOC_FILE); - PrintWriter out = new PrintWriter(new FileWriter(ALLOC_FILE)); - out.println(""); - out.println(""); - out.println(""); - out.println("0.2"); - out.println(""); - out.println(""); - out.close(); + AllocationFileWriter.create() + .addQueue(new AllocationFileQueue.Builder("queue1") + .maxAMShare(0.2).build()) + .writeToFile(ALLOC_FILE); scheduler.init(conf); scheduler.start(); @@ -3800,14 +3683,10 @@ public void testMultipleCompletedEvent() throws Exception { public void testQueueMaxAMShare() throws Exception { conf.set(FairSchedulerConfiguration.ALLOCATION_FILE, ALLOC_FILE); - PrintWriter out = new PrintWriter(new FileWriter(ALLOC_FILE)); - out.println(""); - out.println(""); - out.println(""); - out.println("0.2"); - out.println(""); - out.println(""); - out.close(); + AllocationFileWriter.create() + .addQueue(new AllocationFileQueue.Builder("queue1") + .maxAMShare(0.2).build()) + .writeToFile(ALLOC_FILE); scheduler.init(conf); scheduler.start(); @@ -4030,25 +3909,18 @@ public void testQueueMaxAMShareDefault() throws Exception { conf.set(FairSchedulerConfiguration.ALLOCATION_FILE, ALLOC_FILE); conf.setInt(YarnConfiguration.RM_SCHEDULER_MAXIMUM_ALLOCATION_VCORES, 6); - PrintWriter out = new PrintWriter(new FileWriter(ALLOC_FILE)); - out.println(""); - out.println(""); - out.println(""); - out.println(""); - out.println(""); - out.println("0.4"); - out.println(""); - out.println(""); - out.println("10240 mb 4 vcores"); - out.println(""); - out.println(""); - out.println(""); - out.println(""); - out.println(""); - out.println( - "fair"); - out.println(""); - out.close(); + AllocationFileWriter.create() + .defaultQueueSchedulingPolicy("fair") + .addQueue(new AllocationFileQueue.Builder("queue1").build()) + .addQueue(new AllocationFileQueue.Builder("queue2") + .maxAMShare(0.4f) + .build()) + .addQueue(new AllocationFileQueue.Builder("queue3") + .maxResources("10240 mb 4 vcores") + .build()) + .addQueue(new AllocationFileQueue.Builder("queue4").build()) + .addQueue(new AllocationFileQueue.Builder("queue5").build()) + .writeToFile(ALLOC_FILE); scheduler.init(conf); scheduler.start(); @@ -4191,14 +4063,10 @@ public void testQueueMaxAMShareDefault() throws Exception { public void testQueueMaxAMShareWithContainerReservation() throws Exception { conf.set(FairSchedulerConfiguration.ALLOCATION_FILE, ALLOC_FILE); conf.setFloat(FairSchedulerConfiguration.RESERVABLE_NODES, 1f); - PrintWriter out = new PrintWriter(new FileWriter(ALLOC_FILE)); - out.println(""); - out.println(""); - out.println(""); - out.println("0.5"); - out.println(""); - out.println(""); - out.close(); + AllocationFileWriter.create() + .addQueue(new AllocationFileQueue.Builder("queue1") + .maxAMShare(0.5).build()) + .writeToFile(ALLOC_FILE); scheduler.init(conf); scheduler.start(); @@ -4457,19 +4325,16 @@ public void testMaxRunningAppsHierarchicalQueues() throws Exception { ControlledClock clock = new ControlledClock(); scheduler.setClock(clock); - PrintWriter out = new PrintWriter(new FileWriter(ALLOC_FILE)); - out.println(""); - out.println(""); - out.println(""); - out.println(" 3"); - out.println(" "); - out.println(" "); - out.println(" "); - out.println(" 1"); - out.println(" "); - out.println(""); - out.println(""); - out.close(); + AllocationFileWriter.create() + .addQueue(new AllocationFileQueue.Builder("queue1") + .maxRunningApps(3) + .subQueue(new AllocationFileQueue.Builder("sub1").build()) + .subQueue(new AllocationFileQueue.Builder("sub2").build()) + .subQueue(new AllocationFileQueue.Builder("sub3") + .maxRunningApps(1) + .build()) + .build()) + .writeToFile(ALLOC_FILE); scheduler.init(conf); scheduler.start(); @@ -4581,11 +4446,8 @@ public void testDefaultRuleInitializesProperlyWhenPolicyNotConfigured() conf.setBoolean(FairSchedulerConfiguration.ALLOW_UNDECLARED_POOLS, false); // Create an alloc file with no queue placement policy - PrintWriter out = new PrintWriter(new FileWriter(ALLOC_FILE)); - out.println(""); - out.println(""); - out.println(""); - out.close(); + AllocationFileWriter.create() + .writeToFile(ALLOC_FILE); scheduler.init(conf); scheduler.start(); @@ -5028,16 +4890,13 @@ public void testQueueNameWithTrailingSpace() throws Exception { } @Test - public void testEmptyQueueNameInConfigFile() throws IOException { + public void testEmptyQueueNameInConfigFile() { conf.set(FairSchedulerConfiguration.ALLOCATION_FILE, ALLOC_FILE); // set empty queue name - PrintWriter out = new PrintWriter(new FileWriter(ALLOC_FILE)); - out.println(""); - out.println(""); - out.println(""); - out.println(""); - out.println(""); - out.close(); + AllocationFileWriter.create() + .addQueue(new AllocationFileQueue.Builder("").build()) + .writeToFile(ALLOC_FILE); + try { scheduler.init(conf); Assert.fail("scheduler init should fail because" + @@ -5275,6 +5134,8 @@ public void testRefreshQueuesWhenRMHA() throws Exception { new HAServiceProtocol.StateChangeRequestInfo( HAServiceProtocol.RequestSource.REQUEST_BY_USER); + //ensure ALLOC_FILE contains an 'empty' config + AllocationFileWriter.create().writeToFile(ALLOC_FILE); // 1. start a standby RM, file 'ALLOC_FILE' is empty, so there is no queues MockRM rm1 = new MockRM(conf, null); rm1.init(conf); @@ -5282,14 +5143,10 @@ public void testRefreshQueuesWhenRMHA() throws Exception { rm1.getAdminService().transitionToStandby(requestInfo); // 2. add a new queue "test_queue" - PrintWriter out = new PrintWriter(new FileWriter(ALLOC_FILE)); - out.println(""); - out.println(""); - out.println(""); - out.println(" 3"); - out.println(""); - out.println(""); - out.close(); + AllocationFileWriter.create() + .addQueue(new AllocationFileQueue.Builder("test_queue") + .maxRunningApps(3).build()) + .writeToFile(ALLOC_FILE); conf.set(YarnConfiguration.RM_STORE, MemoryRMStateStore.class.getName()); // 3. start a active RM @@ -5425,16 +5282,12 @@ public void testUpdateDemand() throws IOException { public void testDumpState() throws IOException { conf.set(FairSchedulerConfiguration.ALLOCATION_FILE, ALLOC_FILE); - PrintWriter out = new PrintWriter(new FileWriter(ALLOC_FILE)); - out.println(""); - out.println(""); - out.println(""); - out.println(" "); - out.println(" 1"); - out.println(" "); - out.println(""); - out.println(""); - out.close(); + AllocationFileWriter.create() + .addQueue(new AllocationFileQueue.Builder("parent") + .subQueue(new AllocationFileQueue.Builder("child1") + .weight(1).build()) + .build()) + .writeToFile(ALLOC_FILE); ControlledClock clock = new ControlledClock(); scheduler.setClock(clock); @@ -5613,29 +5466,25 @@ private void testAppRejectedToQueueWithZeroCapacityOfResource(String resource) + "maximum queue resources: .+")); } - private void generateAllocationFileWithZeroResource(String resource) - throws IOException { - PrintWriter out = new PrintWriter(new FileWriter(ALLOC_FILE)); - out.println(""); - out.println(""); - out.println(""); - + private void generateAllocationFileWithZeroResource(String resource) { String resources = ""; if (resource.equals(ResourceInformation.MEMORY_URI)) { resources = "0 mb,2vcores"; } else if (resource.equals(ResourceInformation.VCORES_URI)) { resources = "10000 mb,0vcores"; } - out.println("" + resources + ""); - out.println("" + resources + ""); - out.println("2.0"); - out.println(""); - out.println(""); - out.println("1 mb 1 vcores"); - out.println("0.0"); - out.println(""); - out.println(""); - out.close(); + + AllocationFileWriter.create() + .addQueue(new AllocationFileQueue.Builder("queueA") + .minResources(resources) + .maxResources(resources) + .weight(2.0f) + .build()) + .addQueue(new AllocationFileQueue.Builder("queueB") + .minResources("1 mb 1 vcores") + .weight(0.0f) + .build()) + .writeToFile(ALLOC_FILE); } @Test diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFairSchedulerFairShare.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFairSchedulerFairShare.java index a79aacc196e..c1925fda6f6 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFairSchedulerFairShare.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFairSchedulerFairShare.java @@ -33,6 +33,11 @@ import org.apache.hadoop.yarn.server.resourcemanager.rmnode.RMNode; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.event.AppAttemptRemovedSchedulerEvent; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.event.NodeAddedSchedulerEvent; + +import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair + .allocationfile.AllocationFileQueue; +import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair + .allocationfile.AllocationFileWriter; import org.apache.hadoop.yarn.util.resource.Resources; import org.junit.After; import org.junit.Before; @@ -57,34 +62,29 @@ public void teardown() { conf = null; } - private void createClusterWithQueuesAndOneNode(int mem, String policy) - throws IOException { + private void createClusterWithQueuesAndOneNode(int mem, String policy) { createClusterWithQueuesAndOneNode(mem, 0, policy); } private void createClusterWithQueuesAndOneNode(int mem, int vCores, - String policy) throws IOException { - PrintWriter out = new PrintWriter(new FileWriter(ALLOC_FILE)); - out.println(""); - out.println(""); - out.println(""); - out.println(" "); - out.println(" 8"); - out.println(" "); - out.println(" "); - out.println(" "); - out.println(" "); - out.println(" "); - out.println(" "); - out.println(" 1"); - out.println(" "); - out.println(" "); - out.println(" "); - out.println(""); - out.println("" + policy - + ""); - out.println(""); - out.close(); + String policy) { + AllocationFileWriter.create() + .defaultQueueSchedulingPolicy(policy) + .addQueue(new AllocationFileQueue.Builder("root") + .subQueue(new AllocationFileQueue.Builder("parentA") + .weight(8) + .subQueue(new AllocationFileQueue.Builder("childA1").build()) + .subQueue(new AllocationFileQueue.Builder("childA2").build()) + .subQueue(new AllocationFileQueue.Builder("childA3").build()) + .subQueue(new AllocationFileQueue.Builder("childA4").build()) + .build()) + .subQueue(new AllocationFileQueue.Builder("parentB") + .weight(1) + .subQueue(new AllocationFileQueue.Builder("childB1").build()) + .subQueue(new AllocationFileQueue.Builder("childB2").build()) + .build()) + .build()) + .writeToFile(ALLOC_FILE); resourceManager = new MockRM(conf); resourceManager.start(); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFairSchedulerQueueACLs.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFairSchedulerQueueACLs.java index 32aab5531f8..041b09e3667 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFairSchedulerQueueACLs.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFairSchedulerQueueACLs.java @@ -25,33 +25,34 @@ import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.yarn.conf.YarnConfiguration; import org.apache.hadoop.yarn.server.resourcemanager.QueueACLsTestBase; +import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair + .allocationfile.AllocationFileQueue; +import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair + .allocationfile.AllocationFileWriter; public class TestFairSchedulerQueueACLs extends QueueACLsTestBase { @Override - protected Configuration createConfiguration() throws IOException { + protected Configuration createConfiguration() { FairSchedulerConfiguration fsConf = new FairSchedulerConfiguration(); final String TEST_DIR = new File(System.getProperty("test.build.data", "/tmp")).getAbsolutePath(); final String ALLOC_FILE = new File(TEST_DIR, "test-queues.xml") .getAbsolutePath(); - PrintWriter out = new PrintWriter(new FileWriter(ALLOC_FILE)); - out.println(""); - out.println(""); - out.println(""); - out.println(" "); - out.println(" root_admin "); - out.println(" "); - out.println(" queueA_user,common_user "); - out.println(" queueA_admin "); - out.println(" "); - out.println(" "); - out.println(" queueB_user,common_user "); - out.println(" queueB_admin "); - out.println(" "); - out.println(""); - out.println(""); - out.close(); + + AllocationFileWriter.create() + .addQueue(new AllocationFileQueue.Builder("root") + .aclSubmitApps(" ") + .aclAdministerApps("root_admin ") + .subQueue(new AllocationFileQueue.Builder("queueA") + .aclSubmitApps("queueA_user,common_user ") + .aclAdministerApps("queueA_admin ").build()) + .subQueue(new AllocationFileQueue.Builder("queueB") + .aclSubmitApps("queueB_user,common_user ") + .aclAdministerApps("queueB_admin ").build()) + .build()) + .writeToFile(ALLOC_FILE); + fsConf.set(FairSchedulerConfiguration.ALLOCATION_FILE, ALLOC_FILE); fsConf.setBoolean(YarnConfiguration.YARN_ACL_ENABLE, true); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestQueueManagerRealScheduler.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestQueueManagerRealScheduler.java index 5736f75bbe5..4e4cc7a3d8c 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestQueueManagerRealScheduler.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestQueueManagerRealScheduler.java @@ -18,6 +18,10 @@ package org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair; import org.apache.hadoop.yarn.server.resourcemanager.MockRM; +import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair + .allocationfile.AllocationFileQueue; +import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair + .allocationfile.AllocationFileWriter; import org.junit.After; import org.junit.Before; import org.junit.Test; @@ -38,7 +42,7 @@ @Before public void setup() throws IOException { createConfiguration(); - writeAllocFile(30, 40); + writeAllocFile(30); conf.set(FairSchedulerConfiguration.ALLOCATION_FILE, ALLOC_FILE.getAbsolutePath()); @@ -56,32 +60,20 @@ public void teardown() { } } - private void writeAllocFile(int defaultFairShareTimeout, - int fairShareTimeout) throws IOException { - PrintWriter out = new PrintWriter(new FileWriter(ALLOC_FILE)); - out.println(""); - out.println(""); - out.println(""); - out.println(""); - out.println(""); - out.println(""); - out.println(""); - out.println(""); - out.println("5"); - out.println(""); - out.println(""); - out.println(""); - out.println(""); - out.println(""); - out.println(""); - out.println("15" - + ""); - out.println("" + - + defaultFairShareTimeout + ""); - out.println("" - + fairShareTimeout + ""); - out.println(""); - out.close(); + private void writeAllocFile(int defaultFairShareTimeout) { + AllocationFileWriter.create() + .addQueue(new AllocationFileQueue.Builder("default") + .build()) + .addQueue(new AllocationFileQueue.Builder("queueA").build()) + .addQueue(new AllocationFileQueue.Builder("queueB") + .subQueue(new AllocationFileQueue.Builder("queueB1") + .minSharePreemptionTimeout(5).build()) + .subQueue(new AllocationFileQueue.Builder("queueB2").build()) + .build()) + .addQueue(new AllocationFileQueue.Builder("queueC").build()) + .defaultMinSharePreemptionTimeout(15) + .defaultFairSharePreemptionTimeout(defaultFairShareTimeout) + .writeToFile(ALLOC_FILE.getAbsolutePath()); } @Test @@ -120,7 +112,7 @@ public void testBackwardsCompatiblePreemptionConfiguration() // Lower the fairshare preemption timeouts and verify it is picked // correctly. - writeAllocFile(25, 30); + writeAllocFile(25); scheduler.reinitialize(conf, resourceManager.getRMContext()); assertEquals(25000, queueMgr.getQueue("root") .getFairSharePreemptionTimeout()); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestSchedulingPolicy.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestSchedulingPolicy.java index b016c1b4fb8..db28df43409 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestSchedulingPolicy.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestSchedulingPolicy.java @@ -30,6 +30,10 @@ import org.apache.commons.logging.LogFactory; import org.apache.hadoop.yarn.api.records.Priority; import org.apache.hadoop.yarn.api.records.Resource; +import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair + .allocationfile.AllocationFileQueue; +import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair + .allocationfile.AllocationFileWriter; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.policies.DominantResourceFairnessPolicy; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.policies.FairSharePolicy; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.policies.FifoPolicy; @@ -309,22 +313,16 @@ public boolean isPreemptable() { @Test public void testSchedulingPolicyViolation() throws IOException { conf.set(FairSchedulerConfiguration.ALLOCATION_FILE, ALLOC_FILE); - PrintWriter out = new PrintWriter(new FileWriter(ALLOC_FILE)); - out.println(""); - out.println(""); - out.println(""); - out.println("fair"); - out.println(" "); - out.println(" drf"); - out.println(" "); - out.println(" "); - out.println(" fair"); - out.println(" "); - out.println(""); - out.println("drf" + - ""); - out.println(""); - out.close(); + AllocationFileWriter.create() + .defaultQueueSchedulingPolicy("drf") + .addQueue(new AllocationFileQueue.Builder("root") + .schedulingPolicy("fair") + .subQueue(new AllocationFileQueue.Builder("child1") + .schedulingPolicy("drf").build()) + .subQueue(new AllocationFileQueue.Builder("child2") + .schedulingPolicy("fair").build()) + .build()) + .writeToFile(ALLOC_FILE); scheduler.init(conf); @@ -339,22 +337,16 @@ public void testSchedulingPolicyViolation() throws IOException { + " policy if its parent policy is 'fair'.", dynamicQueue); // Set child1 to 'fair' and child2 to 'drf', the reload the allocation file. - out = new PrintWriter(new FileWriter(ALLOC_FILE)); - out.println(""); - out.println(""); - out.println(""); - out.println("fair"); - out.println(" "); - out.println(" fair"); - out.println(" "); - out.println(" "); - out.println(" drf"); - out.println(" "); - out.println(""); - out.println("drf" + - ""); - out.println(""); - out.close(); + AllocationFileWriter.create() + .defaultQueueSchedulingPolicy("drf") + .addQueue(new AllocationFileQueue.Builder("root") + .schedulingPolicy("fair") + .subQueue(new AllocationFileQueue.Builder("child1") + .schedulingPolicy("fair").build()) + .subQueue(new AllocationFileQueue.Builder("child2") + .schedulingPolicy("drf").build()) + .build()) + .writeToFile(ALLOC_FILE); scheduler.reinitialize(conf, null); child1 = scheduler.getQueueManager().getQueue("child1"); @@ -369,26 +361,21 @@ public void testSchedulingPolicyViolation() throws IOException { } @Test - public void testSchedulingPolicyViolationInTheMiddleLevel() - throws IOException { + public void testSchedulingPolicyViolationInTheMiddleLevel() { conf.set(FairSchedulerConfiguration.ALLOCATION_FILE, ALLOC_FILE); - PrintWriter out = new PrintWriter(new FileWriter(ALLOC_FILE)); - out.println(""); - out.println(""); - out.println(""); - out.println("fair"); - out.println(" "); - out.println(" fair"); - out.println(" "); - out.println(" drf"); - out.println(" "); - out.println(" fair"); - out.println(" "); - out.println(" "); - out.println(" "); - out.println(""); - out.println(""); - out.close(); + AllocationFileWriter.create() + .addQueue(new AllocationFileQueue.Builder("root") + .schedulingPolicy("fair") + .subQueue(new AllocationFileQueue.Builder("level2") + .schedulingPolicy("fair") + .subQueue(new AllocationFileQueue.Builder("level3") + .schedulingPolicy("drf") + .subQueue(new AllocationFileQueue.Builder("leaf") + .schedulingPolicy("fair").build()) + .build()) + .build()) + .build()) + .writeToFile(ALLOC_FILE); scheduler.init(conf); @@ -407,19 +394,16 @@ public void testSchedulingPolicyViolationInTheMiddleLevel() public void testFIFOPolicyOnlyForLeafQueues() throws IOException { conf.set(FairSchedulerConfiguration.ALLOCATION_FILE, ALLOC_FILE); - PrintWriter out = new PrintWriter(new FileWriter(ALLOC_FILE)); - out.println(""); - out.println(""); - out.println(""); - out.println(" "); - out.println(" fifo"); - out.println(" "); - out.println(" fair"); - out.println(" "); - out.println(" "); - out.println(""); - out.println(""); - out.close(); + + AllocationFileWriter.create() + .addQueue(new AllocationFileQueue.Builder("root") + .subQueue(new AllocationFileQueue.Builder("intermediate") + .schedulingPolicy("fifo") + .subQueue(new AllocationFileQueue.Builder("leaf") + .schedulingPolicy("fair").build()) + .build()) + .build()) + .writeToFile(ALLOC_FILE); scheduler.init(conf); @@ -427,19 +411,15 @@ public void testFIFOPolicyOnlyForLeafQueues() assertNull("Queue 'intermediate' should be null since 'fifo' is only for " + "leaf queue.", intermediate); - out = new PrintWriter(new FileWriter(ALLOC_FILE)); - out.println(""); - out.println(""); - out.println(""); - out.println(" "); - out.println(" fair"); - out.println(" "); - out.println(" fifo"); - out.println(" "); - out.println(" "); - out.println(""); - out.println(""); - out.close(); + AllocationFileWriter.create() + .addQueue(new AllocationFileQueue.Builder("root") + .subQueue(new AllocationFileQueue.Builder("intermediate") + .schedulingPolicy("fair") + .subQueue(new AllocationFileQueue.Builder("leaf") + .schedulingPolicy("fifo").build()) + .build()) + .build()) + .writeToFile(ALLOC_FILE); scheduler.reinitialize(conf, null); @@ -453,39 +433,28 @@ public void testFIFOPolicyOnlyForLeafQueues() @Test public void testPolicyReinitilization() throws IOException { conf.set(FairSchedulerConfiguration.ALLOCATION_FILE, ALLOC_FILE); - PrintWriter out = new PrintWriter(new FileWriter(ALLOC_FILE)); - out.println(""); - out.println(""); - out.println(""); - out.println("fair"); - out.println(" "); - out.println(" fair"); - out.println(" "); - out.println(" "); - out.println(" fair"); - out.println(" "); - out.println(""); - out.println(""); - out.close(); + AllocationFileWriter.create() + .addQueue(new AllocationFileQueue.Builder("root") + .schedulingPolicy("fair") + .subQueue(new AllocationFileQueue.Builder("child1") + .schedulingPolicy("fair").build()) + .subQueue(new AllocationFileQueue.Builder("child2") + .schedulingPolicy("fair").build()) + .build()) + .writeToFile(ALLOC_FILE); scheduler.init(conf); // Set child1 to 'drf' which is not allowed, then reload the allocation file - out = new PrintWriter(new FileWriter(ALLOC_FILE)); - out.println(""); - out.println(""); - out.println(""); - out.println("fair"); - out.println(" "); - out.println(" drf"); - out.println(" "); - out.println(" "); - out.println(" fifo"); - out.println(" "); - out.println(""); - out.println(""); - out.close(); - + AllocationFileWriter.create() + .addQueue(new AllocationFileQueue.Builder("root") + .schedulingPolicy("fair") + .subQueue(new AllocationFileQueue.Builder("child1") + .schedulingPolicy("drf").build()) + .subQueue(new AllocationFileQueue.Builder("child2") + .schedulingPolicy("fifo").build()) + .build()) + .writeToFile(ALLOC_FILE); scheduler.reinitialize(conf, null); FSQueue child1 = scheduler.getQueueManager().getQueue("child1"); @@ -498,20 +467,15 @@ public void testPolicyReinitilization() throws IOException { child2.getPolicy() instanceof FairSharePolicy); // Set both child1 and root to 'drf', then reload the allocation file - out = new PrintWriter(new FileWriter(ALLOC_FILE)); - out.println(""); - out.println(""); - out.println(""); - out.println("drf"); - out.println(" "); - out.println(" drf"); - out.println(" "); - out.println(" "); - out.println(" fifo"); - out.println(" "); - out.println(""); - out.println(""); - out.close(); + AllocationFileWriter.create() + .addQueue(new AllocationFileQueue.Builder("root") + .schedulingPolicy("drf") + .subQueue(new AllocationFileQueue.Builder("child1") + .schedulingPolicy("drf").build()) + .subQueue(new AllocationFileQueue.Builder("child2") + .schedulingPolicy("fifo").build()) + .build()) + .writeToFile(ALLOC_FILE); scheduler.reinitialize(conf, null); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/allocationfile/AllocationFileQueue.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/allocationfile/AllocationFileQueue.java index db816130e0e..e9af2805fa5 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/allocationfile/AllocationFileQueue.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/allocationfile/AllocationFileQueue.java @@ -16,69 +16,252 @@ package org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.allocationfile; +import com.google.common.collect.Lists; + import java.io.PrintWriter; import java.io.StringWriter; import java.util.List; -class AllocationFileQueue { - private final AllocationFileQueueProperties properties; +import static org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair + .allocationfile.AllocationFileWriter.addIfPresent; + +public class AllocationFileQueue { + private static final String DEFAULT_TAG_NAME = "queue"; + private static final String LEGACY_TAG_NAME = "pool"; + + private final String queueName; + private final String minResources; + private final String maxResources; + private final String aclAdministerApps; + private final String aclSubmitApps; + private final String aclSubmitReservations; + private final String aclAdministerReservations; + private final String aclListReservations; + private final String schedulingPolicy; + private final Integer maxRunningApps; + private final Double maxAMShare; + private final Integer minSharePreemptionTimeout; + private final String maxChildResources; + private final Integer fairSharePreemptionTimeout; + private final Double fairSharePreemptionThreshold; + private final String maxContainerAllocation; private final List subQueues; + private final Float weight; + private String tagName; + + private final boolean parent; + private final boolean reservation; - AllocationFileQueue(AllocationFileQueueProperties properties, - List subQueues) { - this.properties = properties; - this.subQueues = subQueues; + AllocationFileQueue(Builder builder) { + this.queueName = builder.name; + this.parent = builder.parent; + this.minResources = builder.minResources; + this.maxResources = builder.maxResources; + this.aclAdministerApps = builder.aclAdministerApps; + this.aclSubmitApps = builder.aclSubmitApps; + this.aclSubmitReservations = builder.aclSubmitReservations; + this.aclAdministerReservations = builder.aclAdministerReservations; + this.aclListReservations = builder.aclListReservations; + this.schedulingPolicy = builder.schedulingPolicy; + this.maxRunningApps = builder.maxRunningApps; + this.maxAMShare = builder.maxAMShare; + this.minSharePreemptionTimeout = builder.minSharePreemptionTimeout; + this.maxChildResources = builder.maxChildResources; + this.fairSharePreemptionTimeout = builder.fairSharePreemptionTimeout; + this.fairSharePreemptionThreshold = builder.fairSharePreemptionThreshold; + this.maxContainerAllocation = builder.maxContainerAllocation; + this.weight = builder.weight; + this.reservation = builder.reservation; + this.subQueues = builder.subQueues; + this.tagName = DEFAULT_TAG_NAME; } String render() { + return renderInternal(); + } + + String renderWithLegacyTag() { + this.tagName = LEGACY_TAG_NAME; + return renderInternal(); + } + + private String renderInternal() { StringWriter sw = new StringWriter(); PrintWriter pw = new PrintWriter(sw); printStartTag(pw); - AllocationFileWriter.printQueues(pw, subQueues); - AllocationFileWriter.addIfPresent(pw, "minResources", - properties::getMinResources); - AllocationFileWriter.addIfPresent(pw, "maxResources", - properties::getMaxResources); - AllocationFileWriter.addIfPresent(pw, "aclAdministerApps", - properties::getAclAdministerApps); - AllocationFileWriter.addIfPresent(pw, "aclSubmitApps", - properties::getAclSubmitApps); - AllocationFileWriter.addIfPresent(pw, "schedulingPolicy", - properties::getSchedulingPolicy); - AllocationFileWriter.addIfPresent(pw, "maxRunningApps", - () -> AllocationFileWriter - .createNumberSupplier(properties.getMaxRunningApps())); - AllocationFileWriter.addIfPresent(pw, "maxAMShare", - () -> AllocationFileWriter.createNumberSupplier(properties - .getMaxAMShare())); - AllocationFileWriter.addIfPresent(pw, "minSharePreemptionTimeout", - () -> AllocationFileWriter - .createNumberSupplier(properties.getMinSharePreemptionTimeout())); - AllocationFileWriter.addIfPresent(pw, "maxChildResources", - properties::getMaxChildResources); - AllocationFileWriter.addIfPresent(pw, "fairSharePreemptionTimeout", - () -> AllocationFileWriter - .createNumberSupplier(properties.getFairSharePreemptionTimeout())); - AllocationFileWriter.addIfPresent(pw, "fairSharePreemptionThreshold", - () -> AllocationFileWriter.createNumberSupplier( - properties.getFairSharePreemptionThreshold())); - AllocationFileWriter.addIfPresent(pw, "maxContainerAllocation", - () -> AllocationFileWriter - .createNumberSupplier(properties.getMaxContainerAllocation())); + AllocationFileWriter.printQueues(pw, subQueues, + tagName.equals(LEGACY_TAG_NAME)); + addIfPresent(pw, "minResources", minResources); + addIfPresent(pw, "maxResources", maxResources); + addIfPresent(pw, "aclAdministerApps", aclAdministerApps); + addIfPresent(pw, "aclSubmitApps", aclSubmitApps); + addIfPresent(pw, "aclSubmitReservations", aclSubmitReservations); + addIfPresent(pw, "aclAdministerReservations", aclAdministerReservations); + addIfPresent(pw, "aclListReservations", aclListReservations); + addIfPresent(pw, "schedulingPolicy", schedulingPolicy); + addIfPresent(pw, "maxRunningApps", maxRunningApps); + addIfPresent(pw, "maxAMShare", maxAMShare); + addIfPresent(pw, "minSharePreemptionTimeout", minSharePreemptionTimeout); + addIfPresent(pw, "maxChildResources", maxChildResources); + addIfPresent(pw, "fairSharePreemptionTimeout", fairSharePreemptionTimeout); + addIfPresent(pw, "fairSharePreemptionThreshold", + fairSharePreemptionThreshold); + addIfPresent(pw, "maxContainerAllocation", maxContainerAllocation); + addIfPresent(pw, "weight", weight); + if (reservation) { + pw.println(""); + } printEndTag(pw); pw.close(); return sw.toString(); } private void printStartTag(PrintWriter pw) { - pw.print(""); } private void printEndTag(PrintWriter pw) { - pw.println(""); + pw.println(""); + } + + /** + * Class that can build queues (with subqueues) for testcases. + * The intention of having this class to group the common properties of + * simple queues and subqueues by methods delegating calls to a + * queuePropertiesBuilder instance. + */ + public static class Builder { + String name; + String minResources; + String maxResources; + String aclAdministerApps; + String aclSubmitApps; + String aclSubmitReservations; + String aclAdministerReservations; + String aclListReservations; + String schedulingPolicy; + Integer maxRunningApps; + Double maxAMShare; + Integer minSharePreemptionTimeout; + boolean parent; + String maxChildResources; + Integer fairSharePreemptionTimeout; + Double fairSharePreemptionThreshold; + String maxContainerAllocation; + boolean reservation; + final List subQueues = Lists.newArrayList(); + private Float weight; + + public Builder(String name) { + this.name = name; + } + + public Builder parent(boolean parent) { + this.parent = parent; + return this; + } + + public Builder minResources(String value) { + this.minResources = value; + return this; + } + + public Builder maxResources(String value) { + this.maxResources = value; + return this; + } + + public Builder aclAdministerApps(String value) { + this.aclAdministerApps = value; + return this; + } + + public Builder aclSubmitApps(String value) { + this.aclSubmitApps = value; + return this; + } + + public Builder aclSubmitReservations(String value) { + this.aclSubmitReservations = value; + return this; + } + + public Builder aclAdministerReservations(String value) { + this.aclAdministerReservations = value; + return this; + } + + public Builder aclListReservations(String value) { + this.aclListReservations = value; + return this; + } + + public Builder schedulingPolicy(String value) { + this.schedulingPolicy = value; + return this; + } + + public Builder maxRunningApps(int value) { + this.maxRunningApps = value; + return this; + } + + public Builder maxAMShare(double value) { + this.maxAMShare = value; + return this; + } + + public Builder minSharePreemptionTimeout(int value) { + this.minSharePreemptionTimeout = value; + return this; + } + + public Builder maxChildResources(String value) { + this.maxChildResources = value; + return this; + } + + public Builder fairSharePreemptionTimeout(Integer value) { + this.fairSharePreemptionTimeout = value; + return this; + } + + public Builder fairSharePreemptionThreshold( + double value) { + this.fairSharePreemptionThreshold = value; + return this; + } + + public Builder maxContainerAllocation(String value) { + this.maxContainerAllocation = value; + return this; + } + + public Builder weight(float value) { + this.weight = value; + return this; + } + + public Builder reservation() { + this.reservation = true; + return this; + } + + public Builder subQueue(AllocationFileQueue queue) { + if (queue == null) { + throw new IllegalArgumentException("Subqueue cannot be null!"); + } + subQueues.add(queue); + return this; + } + + public AllocationFileQueue build() { + return new AllocationFileQueue(this); + } } } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/allocationfile/AllocationFileQueueBuilder.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/allocationfile/AllocationFileQueueBuilder.java deleted file mode 100644 index 176024e9b46..00000000000 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/allocationfile/AllocationFileQueueBuilder.java +++ /dev/null @@ -1,121 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * http://www.apache.org/licenses/LICENSE-2.0 - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.allocationfile; - -/** - * Abstract base class for building simple queues and subqueues for testcases. - * Currently there are two concrete types subclassed from this class: - * {@link AllocationFileSimpleQueueBuilder} and - * {@link AllocationFileSubQueueBuilder}. - * The intention of having this class to group the common properties of - * simple queues and subqueues by methods delegating calls to a - * queuePropertiesBuilder instance. - */ -public abstract class AllocationFileQueueBuilder { - final AllocationFileQueueProperties.Builder queuePropertiesBuilder; - - AllocationFileQueueBuilder() { - this.queuePropertiesBuilder = - AllocationFileQueueProperties.Builder.create(); - } - - public AllocationFileQueueBuilder parent(boolean parent) { - this.queuePropertiesBuilder.parent(parent); - return this; - } - - public AllocationFileQueueBuilder minResources(String value) { - this.queuePropertiesBuilder.minResources(value); - return this; - } - - public AllocationFileQueueBuilder maxResources(String value) { - this.queuePropertiesBuilder.maxResources(value); - return this; - } - - public AllocationFileQueueBuilder aclAdministerApps(String value) { - this.queuePropertiesBuilder.aclAdministerApps(value); - return this; - } - - public AllocationFileQueueBuilder aclSubmitApps(String value) { - this.queuePropertiesBuilder.aclSubmitApps(value); - return this; - } - - public AllocationFileQueueBuilder schedulingPolicy(String value) { - this.queuePropertiesBuilder.schedulingPolicy(value); - return this; - } - - public AllocationFileQueueBuilder maxRunningApps(int value) { - this.queuePropertiesBuilder.maxRunningApps(value); - return this; - } - - public AllocationFileQueueBuilder maxAMShare(double value) { - this.queuePropertiesBuilder.maxAMShare(value); - return this; - } - - public AllocationFileQueueBuilder minSharePreemptionTimeout(int value) { - this.queuePropertiesBuilder.minSharePreemptionTimeout(value); - return this; - } - - public AllocationFileQueueBuilder maxChildResources(String value) { - this.queuePropertiesBuilder.maxChildResources(value); - return this; - } - - public AllocationFileQueueBuilder fairSharePreemptionTimeout(Integer value) { - this.queuePropertiesBuilder.fairSharePreemptionTimeout(value); - return this; - } - - public AllocationFileQueueBuilder fairSharePreemptionThreshold( - double value) { - this.queuePropertiesBuilder.fairSharePreemptionThreshold(value); - return this; - } - - public AllocationFileQueueBuilder maxContainerAllocation( - String maxContainerAllocation) { - this.queuePropertiesBuilder.maxContainerAllocation(maxContainerAllocation); - return this; - } - - public AllocationFileQueueBuilder subQueue(String queueName) { - if (this instanceof AllocationFileSimpleQueueBuilder) { - return new AllocationFileSubQueueBuilder( - (AllocationFileSimpleQueueBuilder) this, queueName); - } else { - throw new IllegalStateException( - "subQueue can only be invoked on instances of " - + AllocationFileSimpleQueueBuilder.class); - } - } - - public abstract AllocationFileWriter buildQueue(); - - public abstract AllocationFileSimpleQueueBuilder buildSubQueue(); - - AllocationFileQueueProperties.Builder getqueuePropertiesBuilder() { - return queuePropertiesBuilder; - } -} diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/allocationfile/AllocationFileQueuePlacementPolicy.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/allocationfile/AllocationFileQueuePlacementPolicy.java new file mode 100644 index 00000000000..0d0b9565bff --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/allocationfile/AllocationFileQueuePlacementPolicy.java @@ -0,0 +1,59 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * http://www.apache.org/licenses/LICENSE-2.0 + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.allocationfile; + + +import com.google.common.collect.Lists; + +import java.io.PrintWriter; +import java.io.StringWriter; +import java.util.List; + +public class AllocationFileQueuePlacementPolicy { + private List rules = Lists.newArrayList(); + + public AllocationFileQueuePlacementPolicy addRule( + AllocationFileQueuePlacementRule rule) { + this.rules.add(rule); + return this; + } + + public String render() { + StringWriter sw = new StringWriter(); + PrintWriter pw = new PrintWriter(sw); + addStartTag(pw); + addRules(pw); + addEndTag(pw); + pw.close(); + + return sw.toString(); + } + + private void addStartTag(PrintWriter pw) { + pw.println(""); + } + + private void addRules(PrintWriter pw) { + for (AllocationFileQueuePlacementRule rule : rules) { + pw.println(rule.render()); + } + } + + private void addEndTag(PrintWriter pw) { + pw.println(""); + } +} diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/allocationfile/AllocationFileQueuePlacementRule.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/allocationfile/AllocationFileQueuePlacementRule.java new file mode 100644 index 00000000000..cbc1d872534 --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/allocationfile/AllocationFileQueuePlacementRule.java @@ -0,0 +1,84 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * http://www.apache.org/licenses/LICENSE-2.0 + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.allocationfile; + + +import com.google.common.collect.Lists; + +import java.io.PrintWriter; +import java.io.StringWriter; +import java.util.List; + +public class AllocationFileQueuePlacementRule { + private String name; + private boolean create = true; + private String queue; + private List nestedRules = Lists + .newArrayList(); + + public AllocationFileQueuePlacementRule(String name) { + this.name = name; + } + + public AllocationFileQueuePlacementRule create(boolean create) { + this.create = create; + return this; + } + + public AllocationFileQueuePlacementRule queue(String queue) { + this.queue = queue; + return this; + } + + public AllocationFileQueuePlacementRule addNestedRule( + AllocationFileQueuePlacementRule rule) { + this.nestedRules.add(rule); + return this; + } + + public String render() { + StringWriter sw = new StringWriter(); + PrintWriter pw = new PrintWriter(sw); + addStartTag(pw); + addNestedRules(pw); + addEndTag(pw); + pw.close(); + + return sw.toString(); + } + + private void addStartTag(PrintWriter pw) { + pw.print(""); + } + + private void addNestedRules(PrintWriter pw) { + if (nestedRules != null && !nestedRules.isEmpty()) { + for (AllocationFileQueuePlacementRule rule : nestedRules) { + pw.println(rule.render()); + } + } + } + + private void addEndTag(PrintWriter pw) { + pw.println(""); + } +} diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/allocationfile/AllocationFileQueueProperties.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/allocationfile/AllocationFileQueueProperties.java deleted file mode 100644 index 0a0f330b88c..00000000000 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/allocationfile/AllocationFileQueueProperties.java +++ /dev/null @@ -1,214 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * http://www.apache.org/licenses/LICENSE-2.0 - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.allocationfile; - -/** - * The purpose of this class is to store all properties of a queue. - */ -public class AllocationFileQueueProperties { - private final String queueName; - private final String minResources; - private final String maxResources; - private final String aclAdministerApps; - private final String aclSubmitApps; - private final String schedulingPolicy; - private final Integer maxRunningApps; - private final Double maxAMShare; - private final Integer minSharePreemptionTimeout; - private final Boolean parent; - private final String maxChildResources; - private final Integer fairSharePreemptionTimeout; - private final Double fairSharePreemptionThreshold; - private final String maxContainerAllocation; - - AllocationFileQueueProperties(Builder builder) { - this.queueName = builder.queueName; - this.parent = builder.parent; - this.minResources = builder.minResources; - this.maxResources = builder.maxResources; - this.aclAdministerApps = builder.aclAdministerApps; - this.aclSubmitApps = builder.aclSubmitApps; - this.schedulingPolicy = builder.schedulingPolicy; - this.maxRunningApps = builder.maxRunningApps; - this.maxAMShare = builder.maxAMShare; - this.minSharePreemptionTimeout = builder.minSharePreemptionTimeout; - this.maxChildResources = builder.maxChildResources; - this.fairSharePreemptionTimeout = builder.fairSharePreemptionTimeout; - this.fairSharePreemptionThreshold = builder.fairSharePreemptionThreshold; - this.maxContainerAllocation = builder.maxContainerAllocation; - } - - public String getQueueName() { - return queueName; - } - - public String getMinResources() { - return minResources; - } - - public String getMaxResources() { - return maxResources; - } - - public String getAclAdministerApps() { - return aclAdministerApps; - } - - public String getAclSubmitApps() { - return aclSubmitApps; - } - - public String getSchedulingPolicy() { - return schedulingPolicy; - } - - public Integer getMaxRunningApps() { - return maxRunningApps; - } - - public Double getMaxAMShare() { - return maxAMShare; - } - - public Integer getMinSharePreemptionTimeout() { - return minSharePreemptionTimeout; - } - - public Boolean getParent() { - return parent; - } - - public String getMaxChildResources() { - return maxChildResources; - } - - public Integer getFairSharePreemptionTimeout() { - return fairSharePreemptionTimeout; - } - - public Double getFairSharePreemptionThreshold() { - return fairSharePreemptionThreshold; - } - - public String getMaxContainerAllocation() { - return maxContainerAllocation; - } - - /** - * Builder class for {@link AllocationFileQueueProperties}. - */ - public static final class Builder { - private String queueName; - private Boolean parent = false; - private String minResources; - private String maxResources; - private String aclAdministerApps; - private String aclSubmitApps; - private String schedulingPolicy; - private Integer maxRunningApps; - private Double maxAMShare; - private Integer minSharePreemptionTimeout; - private String maxChildResources; - private Integer fairSharePreemptionTimeout; - private Double fairSharePreemptionThreshold; - private String maxContainerAllocation; - - Builder() { - } - - public static Builder create() { - return new Builder(); - } - - public Builder queueName(String queueName) { - this.queueName = queueName; - return this; - } - - public Builder minResources(String minResources) { - this.minResources = minResources; - return this; - } - - public Builder maxResources(String maxResources) { - this.maxResources = maxResources; - return this; - } - - public Builder aclAdministerApps(String aclAdministerApps) { - this.aclAdministerApps = aclAdministerApps; - return this; - } - - public Builder aclSubmitApps(String aclSubmitApps) { - this.aclSubmitApps = aclSubmitApps; - return this; - } - - public Builder schedulingPolicy(String schedulingPolicy) { - this.schedulingPolicy = schedulingPolicy; - return this; - } - - public Builder maxRunningApps(Integer maxRunningApps) { - this.maxRunningApps = maxRunningApps; - return this; - } - - public Builder maxAMShare(Double maxAMShare) { - this.maxAMShare = maxAMShare; - return this; - } - - public Builder maxContainerAllocation(String maxContainerAllocation) { - this.maxContainerAllocation = maxContainerAllocation; - return this; - } - - public Builder minSharePreemptionTimeout( - Integer minSharePreemptionTimeout) { - this.minSharePreemptionTimeout = minSharePreemptionTimeout; - return this; - } - - public Builder parent(Boolean parent) { - this.parent = parent; - return this; - } - - public Builder maxChildResources(String maxChildResources) { - this.maxChildResources = maxChildResources; - return this; - } - - public Builder fairSharePreemptionTimeout( - Integer fairSharePreemptionTimeout) { - this.fairSharePreemptionTimeout = fairSharePreemptionTimeout; - return this; - } - - public Builder fairSharePreemptionThreshold( - Double fairSharePreemptionThreshold) { - this.fairSharePreemptionThreshold = fairSharePreemptionThreshold; - return this; - } - - public AllocationFileQueueProperties build() { - return new AllocationFileQueueProperties(this); - } - } -} diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/allocationfile/AllocationFileSimpleQueueBuilder.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/allocationfile/AllocationFileSimpleQueueBuilder.java deleted file mode 100644 index 93d100ed35c..00000000000 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/allocationfile/AllocationFileSimpleQueueBuilder.java +++ /dev/null @@ -1,64 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * http://www.apache.org/licenses/LICENSE-2.0 - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.allocationfile; - -import java.util.ArrayList; -import java.util.List; - -/** - * Queue builder that can build a simple queue with its properties. - * Subqueues can be added with {@link #addSubQueue(AllocationFileQueue)}. - */ -public class AllocationFileSimpleQueueBuilder - extends AllocationFileQueueBuilder { - private final AllocationFileWriter allocationFileWriter; - private final List subQueues = new ArrayList<>(); - - AllocationFileSimpleQueueBuilder(AllocationFileWriter allocationFileWriter, - String queueName) { - this.allocationFileWriter = allocationFileWriter; - getqueuePropertiesBuilder().queueName(queueName); - } - - void addSubQueue(AllocationFileQueue queue) { - subQueues.add(queue); - } - - @Override - public AllocationFileWriter buildQueue() { - AllocationFileQueueProperties queueProperties = - getqueuePropertiesBuilder().build(); - AllocationFileQueue queue = - new AllocationFileQueue(queueProperties, subQueues); - - if (allocationFileWriter != null) { - allocationFileWriter.addQueue(queue); - } else { - throw new IllegalStateException( - "allocationFileWriter field has to be set on a " + getClass()); - } - - return allocationFileWriter; - } - - @Override - public AllocationFileSimpleQueueBuilder buildSubQueue() { - throw new IllegalStateException( - "buildSubQueue is not supported in " + getClass()); - } - -} diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/allocationfile/AllocationFileSubQueueBuilder.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/allocationfile/AllocationFileSubQueueBuilder.java deleted file mode 100644 index 728aedcbc82..00000000000 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/allocationfile/AllocationFileSubQueueBuilder.java +++ /dev/null @@ -1,54 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * http://www.apache.org/licenses/LICENSE-2.0 - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.allocationfile; - -import com.google.common.collect.Lists; - - -/** - * Queue builder that can build a subqueue with its properties. - */ -public class AllocationFileSubQueueBuilder extends AllocationFileQueueBuilder { - private AllocationFileSimpleQueueBuilder parentQueueBuilder; - - AllocationFileSubQueueBuilder( - AllocationFileSimpleQueueBuilder parentQueueBuilder, String queueName) { - getqueuePropertiesBuilder().queueName(queueName); - this.parentQueueBuilder = parentQueueBuilder; - } - - @Override - public AllocationFileWriter buildQueue() { - throw new IllegalStateException( - "BuildQueue is not supported in " + getClass()); - } - - public AllocationFileSimpleQueueBuilder buildSubQueue() { - AllocationFileQueueProperties queueProperties = - getqueuePropertiesBuilder().build(); - AllocationFileQueue queue = - new AllocationFileQueue(queueProperties, Lists.newArrayList()); - - if (parentQueueBuilder != null) { - parentQueueBuilder.addSubQueue(queue); - return parentQueueBuilder; - } else { - throw new IllegalStateException( - "parentQueueBuilder field has to be set on a " + getClass()); - } - } -} diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/allocationfile/AllocationFileWriter.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/allocationfile/AllocationFileWriter.java index df1cc53d840..fe14d9d8a40 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/allocationfile/AllocationFileWriter.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/allocationfile/AllocationFileWriter.java @@ -21,7 +21,6 @@ import java.io.PrintWriter; import java.util.ArrayList; import java.util.List; -import java.util.function.Supplier; /** * This class is capable of serializing allocation file data to a file @@ -39,6 +38,10 @@ private String defaultQueueSchedulingPolicy; private List queues = new ArrayList<>(); private UserSettings userSettings; + private boolean useLegacyTagNameForQueues = false; + private String reservationAgent; + private String reservationPolicy; + private AllocationFileQueuePlacementPolicy queuePlacementPolicy; private AllocationFileWriter() { } @@ -47,8 +50,9 @@ public static AllocationFileWriter create() { return new AllocationFileWriter(); } - public AllocationFileSimpleQueueBuilder queue(String queueName) { - return new AllocationFileSimpleQueueBuilder(this, queueName); + public AllocationFileWriter addQueue(AllocationFileQueue queue) { + queues.add(queue); + return this; } public AllocationFileWriter queueMaxAppsDefault(int value) { @@ -92,21 +96,41 @@ public AllocationFileWriter defaultQueueSchedulingPolicy(String value) { return this; } - public UserSettings.Builder userSettings(String username) { - return new UserSettings.Builder(this, username); + public AllocationFileWriter useLegacyTagNameForQueues() { + this.useLegacyTagNameForQueues = true; + return this; + } + + public AllocationFileWriter reservationAgent(String value) { + this.reservationAgent = value; + return this; + } + + public AllocationFileWriter reservationPolicy(String value) { + this.reservationPolicy = value; + return this; } - void addQueue(AllocationFileQueue queue) { - this.queues.add(queue); + public AllocationFileWriter userSettings(UserSettings settings) { + this.userSettings = settings; + return this; } - void setUserSettings(UserSettings userSettings) { - this.userSettings = userSettings; + public AllocationFileWriter queuePlacementPolicy(AllocationFileQueuePlacementPolicy policy) { + this.queuePlacementPolicy = policy; + return this; } - static void printQueues(PrintWriter pw, List queues) { + static void printQueues(PrintWriter pw, List queues, + boolean useLegacyTagName) { for (AllocationFileQueue queue : queues) { - pw.println(queue.render()); + final String queueStr; + if (useLegacyTagName) { + queueStr = queue.renderWithLegacyTag(); + } else { + queueStr = queue.render(); + } + pw.println(queueStr); } } @@ -114,22 +138,18 @@ private void printUserSettings(PrintWriter pw) { pw.println(userSettings.render()); } - static void addIfPresent(PrintWriter pw, String tag, - Supplier supplier) { - if (supplier.get() != null) { - pw.println("<" + tag + ">" + supplier.get() + ""); - } + private void printQueuePlacementPolicy(PrintWriter pw) { + pw.println(queuePlacementPolicy.render()); } - static String createNumberSupplier(Object number) { - if (number != null) { - return number.toString(); + static void addIfPresent(PrintWriter pw, String tag, Object obj) { + if (obj != null) { + pw.println("<" + tag + ">" + obj.toString() + ""); } - return null; } private void writeHeader(PrintWriter pw) { - pw.println(""); + pw.println(""); pw.println(""); } @@ -146,28 +166,31 @@ public void writeToFile(String filename) { } writeHeader(pw); if (!queues.isEmpty()) { - printQueues(pw, queues); + printQueues(pw, queues, useLegacyTagNameForQueues); } if (userSettings != null) { printUserSettings(pw); } - addIfPresent(pw, "queueMaxAppsDefault", - () -> createNumberSupplier(queueMaxAppsDefault)); - addIfPresent(pw, "queueMaxResourcesDefault", - () -> queueMaxResourcesDefault); - addIfPresent(pw, "userMaxAppsDefault", - () -> createNumberSupplier(userMaxAppsDefault)); - addIfPresent(pw, "queueMaxAMShareDefault", - () -> createNumberSupplier(queueMaxAMShareDefault)); + if (queuePlacementPolicy != null) { + printQueuePlacementPolicy(pw); + } + + addIfPresent(pw, "queueMaxAppsDefault", queueMaxAppsDefault); + addIfPresent(pw, "queueMaxResourcesDefault", queueMaxResourcesDefault); + addIfPresent(pw, "userMaxAppsDefault", userMaxAppsDefault); + addIfPresent(pw, "queueMaxAMShareDefault", queueMaxAMShareDefault); addIfPresent(pw, "defaultMinSharePreemptionTimeout", - () -> createNumberSupplier(defaultMinSharePreemptionTimeout)); + defaultMinSharePreemptionTimeout); addIfPresent(pw, "defaultFairSharePreemptionTimeout", - () -> createNumberSupplier(defaultFairSharePreemptionTimeout)); + defaultFairSharePreemptionTimeout); addIfPresent(pw, "defaultFairSharePreemptionThreshold", - () -> createNumberSupplier(defaultFairSharePreemptionThreshold)); + defaultFairSharePreemptionThreshold); addIfPresent(pw, "defaultQueueSchedulingPolicy", - () -> defaultQueueSchedulingPolicy); + defaultQueueSchedulingPolicy); + addIfPresent(pw, "reservation-agent", reservationAgent); + addIfPresent(pw, "reservation-policy", reservationPolicy); + writeFooter(pw); pw.close(); } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/allocationfile/UserSettings.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/allocationfile/UserSettings.java index 7a5656e5f3a..e8457d0e8ec 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/allocationfile/UserSettings.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/allocationfile/UserSettings.java @@ -23,7 +23,7 @@ * Value class that stores user settings and can render data in XML format, * see {@link #render()}. */ -class UserSettings { +public class UserSettings { private final String username; private final Integer maxRunningApps; @@ -36,8 +36,7 @@ public String render() { StringWriter sw = new StringWriter(); PrintWriter pw = new PrintWriter(sw); addStartTag(pw); - AllocationFileWriter.addIfPresent(pw, "maxRunningApps", - () -> AllocationFileWriter.createNumberSupplier(maxRunningApps)); + AllocationFileWriter.addIfPresent(pw, "maxRunningApps", maxRunningApps); addEndTag(pw); pw.close(); @@ -56,12 +55,10 @@ private void addEndTag(PrintWriter pw) { * Builder class for {@link UserSettings} */ public static class Builder { - private final AllocationFileWriter allocationFileWriter; private final String username; private Integer maxRunningApps; - Builder(AllocationFileWriter allocationFileWriter, String username) { - this.allocationFileWriter = allocationFileWriter; + public Builder(String username) { this.username = username; } @@ -70,11 +67,8 @@ public Builder maxRunningApps(int value) { return this; } - public AllocationFileWriter build() { - UserSettings userSettings = new UserSettings(this); - allocationFileWriter.setUserSettings(userSettings); - - return allocationFileWriter; + public UserSettings build() { + return new UserSettings(this); } } } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesAppsModification.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesAppsModification.java index 5d99e0d169d..d370a5951a5 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesAppsModification.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesAppsModification.java @@ -81,6 +81,11 @@ import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacitySchedulerConfiguration; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.FairScheduler; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.FairSchedulerConfiguration; + +import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair + .allocationfile.AllocationFileQueue; +import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair + .allocationfile.AllocationFileWriter; import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.AppPriority; import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.AppQueue; import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.AppState; @@ -202,23 +207,15 @@ public void configureScheduler() { private class FairTestServletModule extends TestServletModule { @Override public void configureScheduler() { - try { - PrintWriter out = new PrintWriter(new FileWriter(FS_ALLOC_FILE)); - out.println(""); - out.println(""); - out.println(""); - out.println(" someuser "); - out.println(" "); - out.println(" someuser "); - out.println(" "); - out.println(" "); - out.println(" someuser "); - out.println(" "); - out.println(""); - out.println(""); - out.close(); - } catch(IOException e) { - } + AllocationFileWriter.create() + .addQueue(new AllocationFileQueue.Builder("root") + .aclAdministerApps("someuser ") + .subQueue(new AllocationFileQueue.Builder("default") + .aclAdministerApps("someuser ").build()) + .subQueue(new AllocationFileQueue.Builder("test") + .aclAdministerApps("someuser ").build()) + .build()) + .writeToFile(FS_ALLOC_FILE); conf.set(FairSchedulerConfiguration.ALLOCATION_FILE, FS_ALLOC_FILE); conf.set(YarnConfiguration.RM_SCHEDULER, FairScheduler.class.getName()); } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesReservation.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesReservation.java index 7e8ba88c88c..9ba643f8cb5 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesReservation.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesReservation.java @@ -52,6 +52,11 @@ import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacitySchedulerConfiguration; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.FairScheduler; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.FairSchedulerConfiguration; + +import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair + .allocationfile.AllocationFileQueue; +import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair + .allocationfile.AllocationFileWriter; import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.ReservationDeleteRequestInfo; import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.ReservationSubmissionRequestInfo; import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.ReservationUpdateRequestInfo; @@ -180,27 +185,17 @@ public void configureScheduler() { private static class FairTestServletModule extends TestServletModule { @Override public void configureScheduler() { - try { - PrintWriter out = new PrintWriter(new FileWriter(FS_ALLOC_FILE)); - out.println(""); - out.println(""); - out.println(""); - out.println(" someuser "); - out.println(" "); - out.println(" someuser "); - out.println(" "); - out.println(" "); - out.println(" "); - out.println(" "); - out.println(" someuser "); - out.println(" "); - out.println(""); - out.println("drf" + - ""); - out.println(""); - out.close(); - } catch (IOException e) { - } + AllocationFileWriter.create() + .defaultQueueSchedulingPolicy("drf") + .addQueue(new AllocationFileQueue.Builder("root") + .aclAdministerApps("someuser ") + .subQueue(new AllocationFileQueue.Builder("default") + .aclAdministerApps("someuser ").build()) + .subQueue(new AllocationFileQueue.Builder("dedicated") + .reservation() + .aclAdministerApps("someuser ").build()) + .build()) + .writeToFile(FS_ALLOC_FILE); conf.set(FairSchedulerConfiguration.ALLOCATION_FILE, FS_ALLOC_FILE); conf.set(YarnConfiguration.RM_SCHEDULER, FairScheduler.class.getName()); }