From 6e211671eae0ebd59a93fd21227ff5aabd6f5609 Mon Sep 17 00:00:00 2001 From: Adam Antal Date: Wed, 27 Nov 2019 15:22:11 +0100 Subject: [PATCH] YARN-5106. Provide a builder interface for FairScheduler allocations for use in tests --- .../impl/TestYarnClientWithReservation.java | 36 +- .../ParameterizedSchedulerTestBase.java | 70 +- .../ReservationACLsTestBase.java | 72 +- .../TestAppManagerWithFairScheduler.java | 119 +-- .../resourcemanager/TestApplicationACLs.java | 41 +- .../ReservationSystemTestUtil.java | 99 +- .../fair/TestAllocationFileLoaderService.java | 365 +++---- .../scheduler/fair/TestAppRunnability.java | 21 +- .../TestApplicationMasterServiceWithFS.java | 35 +- .../scheduler/fair/TestFSAppStarvation.java | 100 +- .../scheduler/fair/TestFSLeafQueue.java | 21 +- .../scheduler/fair/TestFairScheduler.java | 968 ++++++++---------- .../fair/TestFairSchedulerFairShare.java | 50 +- .../fair/TestFairSchedulerPreemption.java | 119 ++- .../fair/TestFairSchedulerQueueACLs.java | 46 +- .../fair/TestQueueManagerRealScheduler.java | 50 +- .../scheduler/fair/TestSchedulingPolicy.java | 206 ++-- .../allocationfile/AllocationFileQueue.java | 274 ++++- .../AllocationFileQueueBuilder.java | 121 --- .../AllocationFileQueuePlacementPolicy.java | 63 ++ .../AllocationFileQueuePlacementRule.java | 88 ++ .../AllocationFileQueueProperties.java | 214 ---- .../AllocationFileSimpleQueueBuilder.java | 64 -- .../AllocationFileSubQueueBuilder.java | 54 - .../allocationfile/AllocationFileWriter.java | 95 +- .../fair/allocationfile/UserSettings.java | 16 +- .../FSConfigConverterTestCommons.java | 47 +- .../TestRMWebServicesAppsModification.java | 33 +- .../webapp/TestRMWebServicesReservation.java | 39 +- 29 files changed, 1548 insertions(+), 1978 deletions(-) delete mode 100644 hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/allocationfile/AllocationFileQueueBuilder.java create mode 100644 hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/allocationfile/AllocationFileQueuePlacementPolicy.java create mode 100644 hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/allocationfile/AllocationFileQueuePlacementRule.java delete mode 100644 hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/allocationfile/AllocationFileQueueProperties.java delete mode 100644 hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/allocationfile/AllocationFileSimpleQueueBuilder.java delete mode 100644 hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/allocationfile/AllocationFileSubQueueBuilder.java diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/api/impl/TestYarnClientWithReservation.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/api/impl/TestYarnClientWithReservation.java index 717647724ac..f9672b31be0 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/api/impl/TestYarnClientWithReservation.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/api/impl/TestYarnClientWithReservation.java @@ -45,6 +45,11 @@ import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacitySchedulerConfiguration; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.FairScheduler; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.FairSchedulerConfiguration; + +import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair + .allocationfile.AllocationFileQueue; +import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair + .allocationfile.AllocationFileWriter; import org.apache.hadoop.yarn.util.Clock; import org.apache.hadoop.yarn.util.UTCClock; import org.junit.Assert; @@ -54,9 +59,7 @@ import org.junit.runners.Parameterized; import java.io.File; -import java.io.FileWriter; import java.io.IOException; -import java.io.PrintWriter; import java.util.Arrays; import java.util.Collection; import java.util.Collections; @@ -136,25 +139,16 @@ private Configuration configureReservationForCapacityScheduler() { private Configuration configureReservationForFairScheduler() { Configuration conf = new Configuration(); - try { - PrintWriter out = new PrintWriter(new FileWriter(FS_ALLOC_FILE)); - out.println(""); - out.println(""); - out.println(""); - out.println(" "); - out.println(" "); - out.println(" "); - // set weight to 10 to make sure this queue get enough steady fair share - out.println(" 10"); - out.println(" "); - out.println(""); - out.println("drf" + - ""); - out.println(""); - out.close(); - } catch (IOException e) { - Assert.fail(e.getMessage()); - } + AllocationFileWriter.create() + .defaultQueueSchedulingPolicy("drf") + .addQueue(new AllocationFileQueue.Builder("root") + .subQueue(new AllocationFileQueue.Builder("default").build()) + .subQueue(new AllocationFileQueue.Builder("dedicated") + .reservation() + .weight(10) + .build()) + .build()) + .writeToFile(FS_ALLOC_FILE); conf.set(FairSchedulerConfiguration.ALLOCATION_FILE, FS_ALLOC_FILE); return conf; diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/ParameterizedSchedulerTestBase.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/ParameterizedSchedulerTestBase.java index 4de16dc88b0..921c93d95a8 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/ParameterizedSchedulerTestBase.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/ParameterizedSchedulerTestBase.java @@ -26,14 +26,18 @@ import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.FairScheduler; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.FairSchedulerConfiguration; + + +import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair + .allocationfile.AllocationFileQueue; +import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair + .allocationfile.AllocationFileWriter; import org.junit.After; import org.junit.runner.RunWith; import org.junit.runners.Parameterized; import java.io.File; -import java.io.FileWriter; import java.io.IOException; -import java.io.PrintWriter; import java.util.Arrays; import java.util.Collection; import java.util.stream.Collectors; @@ -73,43 +77,41 @@ public ParameterizedSchedulerTestBase(SchedulerType type) schedulerType = type; switch (schedulerType) { - case FAIR: - configureFairScheduler(conf); - scheduler = new FairScheduler(); - conf.set(YarnConfiguration.RM_SCHEDULER, - FairScheduler.class.getName()); - break; - case CAPACITY: - scheduler = new CapacityScheduler(); - ((CapacityScheduler)scheduler).setConf(conf); - conf.set(YarnConfiguration.RM_SCHEDULER, - CapacityScheduler.class.getName()); - break; - default: - throw new IllegalArgumentException("Invalid type: " + type); + case FAIR: + configureFairScheduler(conf); + scheduler = new FairScheduler(); + conf.set(YarnConfiguration.RM_SCHEDULER, + FairScheduler.class.getName()); + break; + case CAPACITY: + scheduler = new CapacityScheduler(); + ((CapacityScheduler)scheduler).setConf(conf); + conf.set(YarnConfiguration.RM_SCHEDULER, + CapacityScheduler.class.getName()); + break; + default: + throw new IllegalArgumentException("Invalid type: " + type); } } - protected void configureFairScheduler(YarnConfiguration conf) + protected void configureFairScheduler(YarnConfiguration configuration) throws IOException { // Disable queueMaxAMShare limitation for fair scheduler - PrintWriter out = new PrintWriter(new FileWriter(FS_ALLOC_FILE)); - out.println(""); - out.println(""); - out.println("-1.0"); - out.println("fair"); - out.println(""); - out.println(" drf"); - out.println(" 1.0"); - out.println(" 100"); - out.println(" 120"); - out.println(" .5"); - out.println(""); - out.println(""); - out.close(); - - conf.set(FairSchedulerConfiguration.ALLOCATION_FILE, FS_ALLOC_FILE); - conf.setLong(FairSchedulerConfiguration.UPDATE_INTERVAL_MS, 10); + AllocationFileWriter.create() + .defaultQueueSchedulingPolicy("fair") + .queueMaxAMShareDefault(-1.0) + .addQueue(new AllocationFileQueue.Builder("root") + .schedulingPolicy("drf") + .weight(1.0f) + .fairSharePreemptionTimeout(100) + .minSharePreemptionTimeout(120) + .fairSharePreemptionThreshold(.5) + .build()) + .writeToFile(FS_ALLOC_FILE); + + configuration.set(FairSchedulerConfiguration.ALLOCATION_FILE, + FS_ALLOC_FILE); + configuration.setLong(FairSchedulerConfiguration.UPDATE_INTERVAL_MS, 10); } @After diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/ReservationACLsTestBase.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/ReservationACLsTestBase.java index c8ee00e60bc..9b37693c616 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/ReservationACLsTestBase.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/ReservationACLsTestBase.java @@ -19,9 +19,7 @@ package org.apache.hadoop.yarn.server.resourcemanager; import java.io.File; -import java.io.FileWriter; import java.io.IOException; -import java.io.PrintWriter; import java.util.Arrays; import java.util.Collection; import java.util.Collections; @@ -53,6 +51,12 @@ import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacitySchedulerConfiguration; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.FairScheduler; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.FairSchedulerConfiguration; + + +import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair + .allocationfile.AllocationFileQueue; +import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair + .allocationfile.AllocationFileWriter; import org.apache.hadoop.yarn.server.utils.BuilderUtils; import org.junit.After; import org.junit.Assert; @@ -552,49 +556,35 @@ private static Configuration createCapacitySchedulerConfiguration() { return csConf; } - private static Configuration createFairSchedulerConfiguration() throws - IOException { + private static Configuration createFairSchedulerConfiguration() { FairSchedulerConfiguration fsConf = new FairSchedulerConfiguration(); - final String TEST_DIR = new File(System.getProperty("test.build.data", + final String testDir = new File(System.getProperty("test.build.data", "/tmp")).getAbsolutePath(); - final String ALLOC_FILE = new File(TEST_DIR, "test-queues.xml") + final String allocFile = new File(testDir, "test-queues.xml") .getAbsolutePath(); - PrintWriter out = new PrintWriter(new FileWriter(ALLOC_FILE)); - out.println(""); - out.println(""); - out.println(" drf" + - ""); - out.println(" "); - out.println(" " + - "queueA_user,common_user " + - ""); - out.println(" " + - "queueA_admin " + - ""); - out.println(" common_user "); - out.println(" queueA_user,common_user "); - out.println(" queueA_admin "); - out.println(" "); - out.println(" "); - out.println(" "); - out.println(" queueB_user,common_user "); - out.println(" queueB_admin "); - out.println(" " + - "queueB_user,common_user " + - ""); - out.println(" " + - "queueB_admin " + - ""); - out.println(" common_user "); - out.println(" "); - out.println(" "); - out.println(" "); - out.println(" "); - out.println(" "); - out.println(""); - out.close(); - fsConf.set(FairSchedulerConfiguration.ALLOCATION_FILE, ALLOC_FILE); + + AllocationFileWriter.create() + .defaultQueueSchedulingPolicy("drf") + .addQueue(new AllocationFileQueue.Builder("queueA") + .aclSubmitReservations("queueA_user,common_user ") + .aclAdministerReservations("queueA_admin ") + .aclListReservations("common_user ") + .aclSubmitApps("queueA_user,common_user ") + .aclAdministerApps("queueA_admin ") + .reservation().build()) + .addQueue(new AllocationFileQueue.Builder("queueB") + .aclSubmitReservations("queueB_user,common_user ") + .aclAdministerReservations("queueB_admin ") + .aclListReservations("common_user ") + .aclSubmitApps("queueB_user,common_user ") + .aclAdministerApps("queueB_admin ") + .reservation().build()) + .addQueue(new AllocationFileQueue.Builder("queueC") + .reservation().build()) + .writeToFile(allocFile); + + fsConf.set(FairSchedulerConfiguration.ALLOCATION_FILE, allocFile); fsConf.setBoolean(YarnConfiguration.RM_RESERVATION_SYSTEM_ENABLE, true); fsConf.setBoolean(YarnConfiguration.YARN_ACL_ENABLE, true); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestAppManagerWithFairScheduler.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestAppManagerWithFairScheduler.java index d0dcae0ad4c..57ec60ec19a 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestAppManagerWithFairScheduler.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestAppManagerWithFairScheduler.java @@ -47,6 +47,12 @@ import org.apache.hadoop.yarn.server.resourcemanager.scheduler.ResourceScheduler; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.FairScheduler; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.FairSchedulerConfiguration; + + +import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair + .allocationfile.AllocationFileQueue; +import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair + .allocationfile.AllocationFileWriter; import org.apache.hadoop.yarn.server.resourcemanager.security.ClientToAMTokenSecretManagerInRM; import org.apache.hadoop.yarn.server.security.ApplicationACLsManager; import org.apache.hadoop.yarn.util.Records; @@ -73,13 +79,9 @@ @Before public void setup() throws IOException { // Basic config with one queue (override in test if needed) - PrintWriter out = new PrintWriter(new FileWriter(allocFileName)); - out.println(""); - out.println(""); - out.println(" "); - out.println(" "); - out.println(""); - out.close(); + AllocationFileWriter.create() + .addQueue(new AllocationFileQueue.Builder("test").build()) + .writeToFile(allocFileName); conf.setClass(YarnConfiguration.RM_SCHEDULER, FairScheduler.class, ResourceScheduler.class); @@ -111,19 +113,15 @@ public void testQueueSubmitWithHighQueueContainerSize() YarnConfiguration.DEFAULT_RM_SCHEDULER_MINIMUM_ALLOCATION_MB; // scheduler config with a limited queue - PrintWriter out = new PrintWriter(new FileWriter(allocFileName)); - out.println(""); - out.println(""); - out.println(" "); - out.println(" "); - out.println(" " + maxAlloc + " mb 1 vcores"); - out.println(" "); - out.println(" "); - out.println(" "); - out.println(" "); - out.println(" "); - out.println(""); - out.close(); + AllocationFileWriter.create() + .addQueue(new AllocationFileQueue.Builder("root") + .subQueue(new AllocationFileQueue.Builder("limited") + .maxContainerAllocation(maxAlloc + " mb 1 vcores") + .build()) + .subQueue(new AllocationFileQueue.Builder("unlimited") + .build()) + .build()) + .writeToFile(allocFileName); rmContext.getScheduler().reinitialize(conf, rmContext); ApplicationId appId = MockApps.newAppID(1); @@ -153,25 +151,22 @@ public void testQueueSubmitWithPermissionLimits() conf.set(YarnConfiguration.YARN_ACL_ENABLE, "true"); - PrintWriter out = new PrintWriter(new FileWriter(allocFileName)); - out.println(""); - out.println(""); - out.println(" "); - out.println(" "); - out.println(" "); - out.println(" "); - out.println(" "); - out.println(" "); - out.println(" test "); - out.println(" "); - out.println(" "); - out.println(" "); - out.println(" "); - out.println(" test "); - out.println(" "); - out.println(" "); - out.println(""); - out.close(); + AllocationFileWriter.create() + .addQueue(new AllocationFileQueue.Builder("root") + .aclSubmitApps(" ") + .aclAdministerApps(" ") + .subQueue(new AllocationFileQueue.Builder("noaccess") + .build()) + .subQueue(new AllocationFileQueue.Builder("submitonly") + .aclSubmitApps("test ") + .aclAdministerApps(" ") + .build()) + .subQueue(new AllocationFileQueue.Builder("adminonly") + .aclSubmitApps(" ") + .aclAdministerApps("test ") + .build()) + .build()) + .writeToFile(allocFileName); rmContext.getScheduler().reinitialize(conf, rmContext); ApplicationId appId = MockApps.newAppID(1); @@ -206,17 +201,14 @@ public void testQueueSubmitWithRootPermission() conf.set(YarnConfiguration.YARN_ACL_ENABLE, "true"); - PrintWriter out = new PrintWriter(new FileWriter(allocFileName)); - out.println(""); - out.println(""); - out.println(" "); - out.println(" "); - out.println(" "); - out.println(" "); - out.println(" "); - out.println(" "); - out.println(""); - out.close(); + AllocationFileWriter.create() + .addQueue(new AllocationFileQueue.Builder("root") + .subQueue(new AllocationFileQueue.Builder("noaccess") + .aclSubmitApps(" ") + .aclAdministerApps(" ") + .build()) + .build()) + .writeToFile(allocFileName); rmContext.getScheduler().reinitialize(conf, rmContext); ApplicationId appId = MockApps.newAppID(1); @@ -235,20 +227,19 @@ public void testQueueSubmitWithAutoCreateQueue() conf.set(YarnConfiguration.YARN_ACL_ENABLE, "true"); - PrintWriter out = new PrintWriter(new FileWriter(allocFileName)); - out.println(""); - out.println(""); - out.println(" "); - out.println(" "); - out.println(" "); - out.println(" "); - out.println(" "); - out.println(" "); - out.println(" test "); - out.println(" "); - out.println(" "); - out.println(""); - out.close(); + AllocationFileWriter.create() + .addQueue(new AllocationFileQueue.Builder("root") + .aclSubmitApps(" ") + .aclAdministerApps(" ") + .subQueue(new AllocationFileQueue.Builder("noaccess") + .parent(true) + .build()) + .subQueue(new AllocationFileQueue.Builder("submitonly") + .parent(true) + .aclSubmitApps("test ") + .build()) + .build()) + .writeToFile(allocFileName); rmContext.getScheduler().reinitialize(conf, rmContext); ApplicationId appId = MockApps.newAppID(1); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestApplicationACLs.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestApplicationACLs.java index e952c50353e..c229cb27716 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestApplicationACLs.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestApplicationACLs.java @@ -23,9 +23,7 @@ import static org.mockito.ArgumentMatchers.any; import java.io.File; -import java.io.FileWriter; import java.io.IOException; -import java.io.PrintWriter; import java.net.InetSocketAddress; import java.security.PrivilegedExceptionAction; import java.util.HashMap; @@ -35,6 +33,16 @@ import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMApp; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.FairScheduler; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.FairSchedulerConfiguration; + + +import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair + .allocationfile.AllocationFileQueue; +import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair + .allocationfile.AllocationFileQueuePlacementPolicy; +import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair + .allocationfile.AllocationFileQueuePlacementRule; +import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair + .allocationfile.AllocationFileWriter; import org.junit.After; import org.junit.Assert; import org.slf4j.Logger; @@ -181,26 +189,23 @@ public void tearDown() { } @Override - protected void configureFairScheduler(YarnConfiguration conf) - throws IOException { + protected void configureFairScheduler(YarnConfiguration configuration) { final String testDir = new File(System.getProperty("test.build.data", "/tmp")).getAbsolutePath(); final String allocFile = new File(testDir, "test-queues.xml") .getAbsolutePath(); - PrintWriter out = new PrintWriter(new FileWriter(allocFile)); - out.println(""); - out.println(""); - out.println(""); - out.println(" "); - out.println(" "); - out.println(""); - out.println(""); - out.println(" "); - out.println(" "); - out.println(""); - out.println(""); - out.close(); - conf.set(FairSchedulerConfiguration.ALLOCATION_FILE, allocFile); + + AllocationFileWriter.create() + .addQueue(new AllocationFileQueue.Builder("root") + .subQueue(new AllocationFileQueue.Builder("default").build()) + .build()) + .queuePlacementPolicy(new AllocationFileQueuePlacementPolicy() + .addRule(new AllocationFileQueuePlacementRule("specified") + .create(false)) + .addRule(new AllocationFileQueuePlacementRule("reject"))) + .writeToFile(allocFile); + + configuration.set(FairSchedulerConfiguration.ALLOCATION_FILE, allocFile); } @Test diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/reservation/ReservationSystemTestUtil.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/reservation/ReservationSystemTestUtil.java index 9c138d1279b..4cf8f899321 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/reservation/ReservationSystemTestUtil.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/reservation/ReservationSystemTestUtil.java @@ -24,9 +24,7 @@ import static org.mockito.Mockito.spy; import static org.mockito.Mockito.when; -import java.io.FileWriter; import java.io.IOException; -import java.io.PrintWriter; import java.util.Collections; import java.util.Map; import java.util.Random; @@ -56,6 +54,11 @@ import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacitySchedulerConfiguration; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.event.NodeAddedSchedulerEvent; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.FairScheduler; + +import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair + .allocationfile.AllocationFileQueue; +import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair + .allocationfile.AllocationFileWriter; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.placement.MultiNodeSortingManager; import org.apache.hadoop.yarn.server.resourcemanager.security.ClientToAMTokenSecretManagerInRM; import org.apache.hadoop.yarn.server.resourcemanager.security.NMTokenSecretManagerInRM; @@ -104,62 +107,46 @@ public static void validateReservationQueue( .assertTrue(plan.getSharingPolicy() instanceof CapacityOverTimePolicy); } - public static void setupFSAllocationFile(String allocationFile) - throws IOException { - PrintWriter out = new PrintWriter(new FileWriter(allocationFile)); - out.println(""); - out.println(""); - out.println(""); - out.println("1"); - out.println(""); - out.println(""); - out.println("1"); - out.println(""); - out.println("3"); - out.println(""); - out.println(""); - out.println("7"); - out.println(""); - out.println(""); - out.println(""); - out.println(""); - out.println("8"); - out.println(""); - out.println( - "drf"); - out.println(""); - out.close(); + public static void setupFSAllocationFile(String allocationFile) { + AllocationFileWriter.create() + .defaultQueueSchedulingPolicy("drf") + .addQueue(new AllocationFileQueue.Builder("default") + .weight(1).build()) + .addQueue(new AllocationFileQueue.Builder("a") + .weight(1) + .subQueue(new AllocationFileQueue.Builder("a1") + .weight(3).build()) + .subQueue(new AllocationFileQueue.Builder("a2") + .weight(7).build()) + .build()) + .addQueue(new AllocationFileQueue.Builder("dedicated") + .weight(8) + .reservation() + .build()) + .writeToFile(allocationFile); } - public static void updateFSAllocationFile(String allocationFile) - throws IOException { - PrintWriter out = new PrintWriter(new FileWriter(allocationFile)); - out.println(""); - out.println(""); - out.println(""); - out.println("5"); - out.println(""); - out.println(""); - out.println("5"); - out.println(""); - out.println("3"); - out.println(""); - out.println(""); - out.println("7"); - out.println(""); - out.println(""); - out.println(""); - out.println(""); - out.println("10"); - out.println(""); - out.println(""); - out.println(""); - out.println("80"); - out.println(""); - out.println( - "drf"); - out.println(""); - out.close(); + public static void updateFSAllocationFile(String allocationFile) { + AllocationFileWriter.create() + .defaultQueueSchedulingPolicy("drf") + .addQueue(new AllocationFileQueue.Builder("default") + .weight(5).build()) + .addQueue(new AllocationFileQueue.Builder("a") + .weight(5) + .subQueue(new AllocationFileQueue.Builder("a1") + .weight(3).build()) + .subQueue(new AllocationFileQueue.Builder("a2") + .weight(7).build()) + .build()) + .addQueue(new AllocationFileQueue.Builder("dedicated") + .weight(10) + .reservation() + .build()) + .addQueue(new AllocationFileQueue.Builder("reservation") + .weight(80) + .reservation() + .build()) + .writeToFile(allocationFile); } public static FairScheduler setupFairScheduler(RMContext rmContext, diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestAllocationFileLoaderService.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestAllocationFileLoaderService.java index a02ef551e12..c64cc27cffb 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestAllocationFileLoaderService.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestAllocationFileLoaderService.java @@ -34,24 +34,26 @@ import org.apache.hadoop.yarn.server.resourcemanager.placement.SpecifiedPlacementRule; import org.apache.hadoop.yarn.server.resourcemanager.placement.UserPlacementRule; import org.apache.hadoop.yarn.server.resourcemanager.reservation.ReservationSchedulerConfiguration; + +import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.allocationfile.AllocationFileQueue; +import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.allocationfile.AllocationFileQueuePlacementPolicy; +import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.allocationfile.AllocationFileQueuePlacementRule; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.allocationfile.AllocationFileWriter; +import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.allocationfile.UserSettings; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.policies.DominantResourceFairnessPolicy; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.policies.FairSharePolicy; import org.apache.hadoop.yarn.util.ControlledClock; import org.apache.hadoop.yarn.util.SystemClock; import org.apache.hadoop.yarn.util.resource.CustomResourceTypesConfigurationProvider; import org.apache.hadoop.yarn.util.resource.Resources; +import org.junit.After; import org.junit.Before; import org.junit.Test; + import java.io.File; -import java.io.FileOutputStream; -import java.io.FileWriter; import java.io.IOException; -import java.io.OutputStreamWriter; -import java.io.PrintWriter; import java.net.URISyntaxException; import java.net.URL; -import java.nio.charset.StandardCharsets; import java.util.List; import java.util.Map; import java.util.Set; @@ -97,6 +99,11 @@ public void setup() { when(scheduler.getRMContext()).thenReturn(rmContext); } + @After + public void teardown() { + new File(ALLOC_FILE).delete(); + } + @Test public void testGetAllocationFileFromFileSystem() throws IOException, URISyntaxException { @@ -152,18 +159,13 @@ public void testGetAllocationFileFromClasspath() { @Test (timeout = 10000) public void testReload() throws Exception { - PrintWriter out = new PrintWriter(new FileWriter(ALLOC_FILE)); - out.println(""); - out.println(""); - out.println(" "); - out.println(" 1"); - out.println(" "); - out.println(" "); - out.println(" "); - out.println(" "); - out.println(" "); - out.println(""); - out.close(); + AllocationFileWriter.create() + .addQueue(new AllocationFileQueue.Builder("queueA") + .maxRunningApps(1).build()) + .addQueue(new AllocationFileQueue.Builder("queueB").build()) + .queuePlacementPolicy(new AllocationFileQueuePlacementPolicy() + .addRule(new AllocationFileQueuePlacementRule("default"))) + .writeToFile(ALLOC_FILE); ControlledClock clock = new ControlledClock(); clock.setTime(0); @@ -195,20 +197,15 @@ public void testReload() throws Exception { confHolder.allocConf = null; // Modify file and advance the clock - out = new PrintWriter(new FileWriter(ALLOC_FILE)); - out.println(""); - out.println(""); - out.println(" "); - out.println(" 3"); - out.println(" "); - out.println(" "); - out.println(" "); - out.println(" "); - out.println(" "); - out.println(" "); - out.println(" "); - out.println(""); - out.close(); + AllocationFileWriter.create() + .addQueue(new AllocationFileQueue.Builder("queueB") + .maxRunningApps(3).build()) + .queuePlacementPolicy(new AllocationFileQueuePlacementPolicy() + .addRule(new AllocationFileQueuePlacementRule("specified")) + .addRule(new AllocationFileQueuePlacementRule("nestedUserQueue") + .addNestedRule(new AllocationFileQueuePlacementRule( + "primaryGroup")))) + .writeToFile(ALLOC_FILE); clock.tickMsec(System.currentTimeMillis() + AllocationFileLoaderService.ALLOC_RELOAD_WAIT_MS + 10000); @@ -242,57 +239,56 @@ public void testAllocationFileParsing() throws Exception { AllocationFileLoaderService allocLoader = new AllocationFileLoaderService(scheduler); - AllocationFileWriter - .create() + AllocationFileWriter.create() // Give queue A a minimum of 1024 M - .queue("queueA") - .minResources("1024mb,0vcores") - .maxResources("2048mb,10vcores") - .buildQueue() + .addQueue(new AllocationFileQueue.Builder("queueA") + .minResources("1024mb,0vcores") + .maxResources("2048mb,10vcores") + .build()) // Give queue B a minimum of 2048 M - .queue("queueB") + .addQueue(new AllocationFileQueue.Builder("queueB") .minResources("2048mb,0vcores") .maxResources("5120mb,110vcores") .aclAdministerApps("alice,bob admins") .schedulingPolicy("fair") - .buildQueue() + .build()) // Give queue C no minimum - .queue("queueC") - .minResources("5120mb,0vcores") - .aclSubmitApps("alice,bob admins") - .buildQueue() + .addQueue(new AllocationFileQueue.Builder("queueC") + .minResources("5120mb,0vcores") + .aclSubmitApps("alice,bob admins") + .build()) // Give queue D a limit of 3 running apps and 0.4f maxAMShare - .queue("queueD") - .maxRunningApps(3) - .maxAMShare(0.4) - .buildQueue() + .addQueue(new AllocationFileQueue.Builder("queueD") + .maxRunningApps(3) + .maxAMShare(0.4) + .build()) // Give queue E a preemption timeout of one minute - .queue("queueE") - .minSharePreemptionTimeout(60) - .buildQueue() + .addQueue(new AllocationFileQueue.Builder("queueE") + .minSharePreemptionTimeout(60) + .build()) // Make queue F a parent queue without configured leaf queues // using the 'type' attribute - .queue("queueF") - .parent(true) - .maxChildResources("2048mb,64vcores") - .buildQueue() - .queue("queueG") - .maxChildResources("2048mb,64vcores") - .fairSharePreemptionTimeout(120) - .minSharePreemptionTimeout(50) - .fairSharePreemptionThreshold(0.6) - .maxContainerAllocation( + .addQueue(new AllocationFileQueue.Builder("queueF") + .parent(true) + .maxChildResources("2048mb,64vcores") + .build()) + .addQueue(new AllocationFileQueue.Builder("queueG") + .maxChildResources("2048mb,64vcores") + .fairSharePreemptionTimeout(120) + .minSharePreemptionTimeout(50) + .fairSharePreemptionThreshold(0.6) + .maxContainerAllocation( "vcores=16, memory-mb=512, " + A_CUSTOM_RESOURCE + "=10") // Create hierarchical queues G,H, with different min/fair // share preemption timeouts and preemption thresholds. // Also add a child default to make sure it doesn't impact queue H. - .subQueue("queueH") - .fairSharePreemptionTimeout(180) - .minSharePreemptionTimeout(40) - .fairSharePreemptionThreshold(0.7) - .maxContainerAllocation("1024mb,8vcores") - .buildSubQueue() - .buildQueue() + .subQueue(new AllocationFileQueue.Builder("queueH") + .fairSharePreemptionTimeout(180) + .minSharePreemptionTimeout(40) + .fairSharePreemptionThreshold(0.7) + .maxContainerAllocation("1024mb,8vcores") + .build()) + .build()) // Set default limit of apps per queue to 15 .queueMaxAppsDefault(15) // Set default limit of max resource per queue to 4G and 100 cores @@ -310,9 +306,9 @@ public void testAllocationFileParsing() throws Exception { // Set default scheduling policy to DRF .defaultQueueSchedulingPolicy("drf") // Give user1 a limit of 10 jobs - .userSettings("user1") + .userSettings(new UserSettings.Builder("user1") .maxRunningApps(10) - .build() + .build()) .writeToFile(ALLOC_FILE); allocLoader.init(conf); @@ -488,48 +484,51 @@ public void testBackwardsCompatibleAllocationFileParsing() throws Exception { AllocationFileLoaderService allocLoader = new AllocationFileLoaderService(scheduler); - PrintWriter out = new PrintWriter(new FileWriter(ALLOC_FILE)); - out.println(""); - out.println(""); - // Give queue A a minimum of 1024 M - out.println(""); - out.println("1024mb,0vcores"); - out.println(""); - // Give queue B a minimum of 2048 M - out.println(""); - out.println("2048mb,0vcores"); - out.println("alice,bob admins"); - out.println(""); - // Give queue C no minimum - out.println(""); - out.println("alice,bob admins"); - out.println(""); - // Give queue D a limit of 3 running apps - out.println(""); - out.println("3"); - out.println(""); - // Give queue E a preemption timeout of one minute and 0.3f threshold - out.println(""); - out.println("60"); - out.println("0.3"); - out.println(""); - // Set default limit of apps per queue to 15 - out.println("15"); - // Set default limit of apps per user to 5 - out.println("5"); - // Give user1 a limit of 10 jobs - out.println(""); - out.println("10"); - out.println(""); - // Set default min share preemption timeout to 2 minutes - out.println("120" - + ""); - // Set fair share preemption timeout to 5 minutes - out.println("300"); - // Set default fair share preemption threshold to 0.6f - out.println("0.6"); - out.println(""); - out.close(); + AllocationFileWriter.create() + .useLegacyTagNameForQueues() + // Give queue A a minimum of 1024 M + .addQueue(new AllocationFileQueue.Builder("queueA") + .minResources("1024mb,0vcores") + .build()) + // Give queue B a minimum of 2048 M + .addQueue(new AllocationFileQueue.Builder("queueB") + .minResources("2048mb,0vcores") + .aclAdministerApps("alice,bob admins") + .build()) + // Give queue C no minimum + .addQueue(new AllocationFileQueue.Builder("queueC") + .aclAdministerApps("alice,bob admins") + .build()) + // Give queue D a limit of 3 running apps + .addQueue(new AllocationFileQueue.Builder("queueD") + .maxRunningApps(3) + .build()) + // Give queue E a preemption timeout of one minute and 0.3f threshold + .addQueue(new AllocationFileQueue.Builder("queueE") + .minSharePreemptionTimeout(60) + .fairSharePreemptionThreshold(0.3) + .build()) + // Set default limit of apps per queue to 15 + .queueMaxAppsDefault(15) + // Set default limit of apps per user to 5 + .userMaxAppsDefault(5) + // Set default limit of max resource per queue to 4G and 100 cores + .queueMaxResourcesDefault("4096mb,100vcores") + // Set default limit of AMResourceShare to 0.5f + .queueMaxAMShareDefault(0.5) + // Set default min share preemption timeout to 2 minutes + .defaultMinSharePreemptionTimeout(120) + // Set default fair share preemption timeout to 5 minutes + .defaultFairSharePreemptionTimeout(300) + // Set default fair share preemption threshold to 0.6 + .defaultFairSharePreemptionThreshold(0.6) + // Set default scheduling policy to DRF + .defaultQueueSchedulingPolicy("drf") + // Give user1 a limit of 10 jobs + .userSettings(new UserSettings.Builder("user1") + .maxRunningApps(10) + .build()) + .writeToFile(ALLOC_FILE); allocLoader.init(conf); ReloadListener confHolder = new ReloadListener(); @@ -602,11 +601,7 @@ public void testSimplePlacementPolicyFromConf() throws Exception { conf.setBoolean(FairSchedulerConfiguration.ALLOW_UNDECLARED_POOLS, false); conf.setBoolean(FairSchedulerConfiguration.USER_AS_DEFAULT_QUEUE, false); - PrintWriter out = new PrintWriter(new FileWriter(ALLOC_FILE)); - out.println(""); - out.println(""); - out.println(""); - out.close(); + AllocationFileWriter.create().writeToFile(ALLOC_FILE); AllocationFileLoaderService allocLoader = new AllocationFileLoaderService(scheduler); @@ -632,15 +627,10 @@ public void testSimplePlacementPolicyFromConf() throws Exception { public void testQueueAlongsideRoot() throws Exception { conf.set(FairSchedulerConfiguration.ALLOCATION_FILE, ALLOC_FILE); - PrintWriter out = new PrintWriter(new FileWriter(ALLOC_FILE)); - out.println(""); - out.println(""); - out.println(""); - out.println(""); - out.println(""); - out.println(""); - out.println(""); - out.close(); + AllocationFileWriter.create() + .addQueue(new AllocationFileQueue.Builder("root").build()) + .addQueue(new AllocationFileQueue.Builder("other").build()) + .writeToFile(ALLOC_FILE); AllocationFileLoaderService allocLoader = new AllocationFileLoaderService(scheduler); @@ -658,13 +648,9 @@ public void testQueueAlongsideRoot() throws Exception { public void testQueueNameContainingPeriods() throws Exception { conf.set(FairSchedulerConfiguration.ALLOCATION_FILE, ALLOC_FILE); - PrintWriter out = new PrintWriter(new FileWriter(ALLOC_FILE)); - out.println(""); - out.println(""); - out.println(""); - out.println(""); - out.println(""); - out.close(); + AllocationFileWriter.create() + .addQueue(new AllocationFileQueue.Builder("parent1.child").build()) + .writeToFile(ALLOC_FILE); AllocationFileLoaderService allocLoader = new AllocationFileLoaderService(scheduler); @@ -682,13 +668,9 @@ public void testQueueNameContainingPeriods() throws Exception { public void testQueueNameContainingOnlyWhitespace() throws Exception { conf.set(FairSchedulerConfiguration.ALLOCATION_FILE, ALLOC_FILE); - PrintWriter out = new PrintWriter(new FileWriter(ALLOC_FILE)); - out.println(""); - out.println(""); - out.println(""); - out.println(""); - out.println(""); - out.close(); + AllocationFileWriter.create() + .addQueue(new AllocationFileQueue.Builder(" ").build()) + .writeToFile(ALLOC_FILE); AllocationFileLoaderService allocLoader = new AllocationFileLoaderService(scheduler); @@ -702,15 +684,12 @@ public void testQueueNameContainingOnlyWhitespace() throws Exception { public void testParentTagWithReservation() throws Exception { conf.set(FairSchedulerConfiguration.ALLOCATION_FILE, ALLOC_FILE); - PrintWriter out = new PrintWriter(new FileWriter(ALLOC_FILE)); - out.println(""); - out.println(""); - out.println(""); - out.println(""); - out.println(""); - out.println(""); - out.println(""); - out.close(); + AllocationFileWriter.create() + .addQueue(new AllocationFileQueue.Builder("parent") + .parent(true) + .reservation() + .build()) + .writeToFile(ALLOC_FILE); AllocationFileLoaderService allocLoader = new AllocationFileLoaderService(scheduler); @@ -731,17 +710,13 @@ public void testParentTagWithReservation() throws Exception { public void testParentWithReservation() throws Exception { conf.set(FairSchedulerConfiguration.ALLOCATION_FILE, ALLOC_FILE); - PrintWriter out = new PrintWriter(new FileWriter(ALLOC_FILE)); - out.println(""); - out.println(""); - out.println(""); - out.println(""); - out.println(""); - out.println(" "); - out.println(" "); - out.println(""); - out.println(""); - out.close(); + AllocationFileWriter.create() + .addQueue(new AllocationFileQueue.Builder("parent") + .parent(true) + .subQueue(new AllocationFileQueue.Builder("child").build()) + .reservation() + .build()) + .writeToFile(ALLOC_FILE); AllocationFileLoaderService allocLoader = new AllocationFileLoaderService(scheduler); @@ -762,15 +737,12 @@ public void testParentWithReservation() throws Exception { public void testParentTagWithChild() throws Exception { conf.set(FairSchedulerConfiguration.ALLOCATION_FILE, ALLOC_FILE); - PrintWriter out = new PrintWriter(new FileWriter(ALLOC_FILE)); - out.println(""); - out.println(""); - out.println(""); - out.println(" "); - out.println(" "); - out.println(""); - out.println(""); - out.close(); + AllocationFileWriter.create() + .addQueue(new AllocationFileQueue.Builder("parent") + .parent(true) + .subQueue(new AllocationFileQueue.Builder("child").build()) + .build()) + .writeToFile(ALLOC_FILE); AllocationFileLoaderService allocLoader = new AllocationFileLoaderService(scheduler); @@ -794,14 +766,9 @@ public void testParentTagWithChild() throws Exception { public void testQueueNameContainingNBWhitespace() throws Exception { conf.set(FairSchedulerConfiguration.ALLOCATION_FILE, ALLOC_FILE); - PrintWriter out = new PrintWriter(new OutputStreamWriter( - new FileOutputStream(ALLOC_FILE), StandardCharsets.UTF_8)); - out.println(""); - out.println(""); - out.println(""); - out.println(""); - out.println(""); - out.close(); + AllocationFileWriter.create() + .addQueue(new AllocationFileQueue.Builder("\u00a0").build()) + .writeToFile(ALLOC_FILE); AllocationFileLoaderService allocLoader = new AllocationFileLoaderService(scheduler); @@ -818,13 +785,9 @@ public void testQueueNameContainingNBWhitespace() throws Exception { public void testDefaultQueueSchedulingModeIsFIFO() throws Exception { conf.set(FairSchedulerConfiguration.ALLOCATION_FILE, ALLOC_FILE); - PrintWriter out = new PrintWriter(new FileWriter(ALLOC_FILE)); - out.println(""); - out.println(""); - out.println("fifo" + - ""); - out.println(""); - out.close(); + AllocationFileWriter.create() + .defaultQueueSchedulingPolicy("fifo") + .writeToFile(ALLOC_FILE); AllocationFileLoaderService allocLoader = new AllocationFileLoaderService(scheduler); @@ -838,19 +801,14 @@ public void testDefaultQueueSchedulingModeIsFIFO() throws Exception { public void testReservableQueue() throws Exception { conf.set(FairSchedulerConfiguration.ALLOCATION_FILE, ALLOC_FILE); - PrintWriter out = new PrintWriter(new FileWriter(ALLOC_FILE)); - out.println(""); - out.println(""); - out.println(""); - out.println(""); - out.println(""); - out.println(""); - out.println(""); - out.println(""); - out.println("DummyAgentName"); - out.println("AnyAdmissionPolicy"); - out.println(""); - out.close(); + AllocationFileWriter.create() + .addQueue(new AllocationFileQueue.Builder("reservable") + .reservation() + .build()) + .addQueue(new AllocationFileQueue.Builder("other").build()) + .reservationAgent("DummyAgentName") + .reservationPolicy("AnyAdmissionPolicy") + .writeToFile(ALLOC_FILE); AllocationFileLoaderService allocLoader = new AllocationFileLoaderService(scheduler); @@ -900,15 +858,12 @@ public void testReservableCannotBeCombinedWithDynamicUserQueue() throws Exception { conf.set(FairSchedulerConfiguration.ALLOCATION_FILE, ALLOC_FILE); - PrintWriter out = new PrintWriter(new FileWriter(ALLOC_FILE)); - out.println(""); - out.println(""); - out.println(""); - out.println(""); - out.println(""); - out.println(""); - out.println(""); - out.close(); + AllocationFileWriter.create() + .addQueue(new AllocationFileQueue.Builder("notboth") + .parent(true) + .reservation() + .build()) + .writeToFile(ALLOC_FILE); AllocationFileLoaderService allocLoader = new AllocationFileLoaderService(scheduler); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestAppRunnability.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestAppRunnability.java index b7ef4713819..9777e141970 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestAppRunnability.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestAppRunnability.java @@ -24,9 +24,7 @@ import static org.junit.Assert.assertTrue; import java.io.File; -import java.io.FileWriter; import java.io.IOException; -import java.io.PrintWriter; import org.apache.hadoop.metrics2.lib.DefaultMetricsSystem; import org.apache.hadoop.yarn.api.records.ApplicationAttemptId; @@ -43,6 +41,12 @@ import org.apache.hadoop.yarn.server.resourcemanager.scheduler.event.AppAttemptRemovedSchedulerEvent; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.event.NodeAddedSchedulerEvent; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.event.NodeUpdateSchedulerEvent; + + +import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair + .allocationfile.AllocationFileQueue; +import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair + .allocationfile.AllocationFileWriter; import org.apache.hadoop.yarn.util.resource.Resources; import org.junit.After; import org.junit.Before; @@ -164,17 +168,14 @@ public void testPreemptionVariablesForQueueCreatedRuntime() throws Exception { } @Test - public void testDontAllowUndeclaredPools() throws Exception { + public void testDontAllowUndeclaredPools() { conf.setBoolean(FairSchedulerConfiguration.ALLOW_UNDECLARED_POOLS, false); conf.set(FairSchedulerConfiguration.ALLOCATION_FILE, ALLOC_FILE); - PrintWriter out = new PrintWriter(new FileWriter(ALLOC_FILE)); - out.println(""); - out.println(""); - out.println(""); - out.println(""); - out.println(""); - out.close(); + AllocationFileWriter.create() + .addQueue(new AllocationFileQueue.Builder("jerry").build()) + .writeToFile(ALLOC_FILE); + // Restarting resource manager since the file location and content is // changed. resourceManager.stop(); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestApplicationMasterServiceWithFS.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestApplicationMasterServiceWithFS.java index d7ec1621910..b6f1b8cfc02 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestApplicationMasterServiceWithFS.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestApplicationMasterServiceWithFS.java @@ -19,9 +19,6 @@ package org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair; import java.io.File; -import java.io.FileWriter; -import java.io.IOException; -import java.io.PrintWriter; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -36,6 +33,12 @@ import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMApp; import org.apache.hadoop.yarn.server.resourcemanager.rmapp.attempt.RMAppAttempt; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.ResourceScheduler; + + +import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair + .allocationfile.AllocationFileQueue; +import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair + .allocationfile.AllocationFileWriter; import org.junit.AfterClass; import org.junit.Assert; import org.junit.BeforeClass; @@ -56,7 +59,7 @@ private static YarnConfiguration configuration; @BeforeClass - public static void setup() throws IOException { + public static void setup() { String allocFile = GenericTestUtils.getTestDir(TEST_FOLDER).getAbsolutePath(); @@ -65,21 +68,15 @@ public static void setup() throws IOException { ResourceScheduler.class); configuration.set(FairSchedulerConfiguration.ALLOCATION_FILE, allocFile); - PrintWriter out = new PrintWriter(new FileWriter(allocFile)); - out.println(""); - out.println(""); - out.println(" "); - out.println( - " 2048 mb 1 vcores"); - out.println(" "); - out.println(" "); - out.println( - " 3072 mb 1 vcores"); - out.println(" "); - out.println(" "); - out.println(" "); - out.println(""); - out.close(); + AllocationFileWriter.create() + .addQueue(new AllocationFileQueue.Builder("queueA") + .maxContainerAllocation("2048 mb 1 vcores") + .build()) + .addQueue(new AllocationFileQueue.Builder("queueB") + .maxContainerAllocation("3072 mb 1 vcores") + .build()) + .addQueue(new AllocationFileQueue.Builder("queueC").build()) + .writeToFile(allocFile); } @AfterClass diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFSAppStarvation.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFSAppStarvation.java index 9665f9a7bb0..b5d233b8722 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFSAppStarvation.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFSAppStarvation.java @@ -22,6 +22,11 @@ import org.apache.hadoop.yarn.server.resourcemanager.MockRM; import org.apache.hadoop.yarn.server.resourcemanager.rmnode.RMNode; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.event.NodeUpdateSchedulerEvent; + +import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair + .allocationfile.AllocationFileQueue; +import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair + .allocationfile.AllocationFileWriter; import org.apache.hadoop.yarn.util.ControlledClock; import org.junit.After; @@ -33,12 +38,10 @@ import org.junit.Test; import java.io.File; -import java.io.FileWriter; import java.io.IOException; -import java.io.PrintWriter; /** - * Test class to verify identification of app starvation + * Test class to verify identification of app starvation. */ public class TestFSAppStarvation extends FairSchedulerTestBase { @@ -187,52 +190,40 @@ private void setupClusterAndSubmitJobs() throws Exception { * 4. Submit an app that uses up all resources on the cluster */ private void setupStarvedCluster() throws IOException { - PrintWriter out = new PrintWriter(new FileWriter(ALLOC_FILE)); - out.println(""); - out.println(""); - - // Default queue - out.println(""); - out.println(""); - - // Queue with preemption disabled - out.println(""); - out.println("0" + - ""); - out.println(""); - - // Queue with minshare preemption enabled - out.println(""); - out.println("0" + - ""); - out.println("0" + - ""); - out.println("2048mb,2vcores"); - out.println(""); - - // FAIR queue with fairshare preemption enabled - out.println(""); - out.println("1" + - ""); - out.println("0" + - ""); - out.println("fair"); - addChildQueue(out, "fair"); - out.println(""); - - // DRF queue with fairshare preemption enabled - out.println(""); - out.println("1" + - ""); - out.println("0" + - ""); - out.println("drf"); - addChildQueue(out, "drf"); - out.println(""); - out.println("drf" + - ""); - out.println(""); - out.close(); + AllocationFileWriter.create() + .defaultQueueSchedulingPolicy("drf") + // Default queue + .addQueue(new AllocationFileQueue.Builder("default").build()) + // Queue with preemption disabled + .addQueue(new AllocationFileQueue.Builder("no-preemption") + .fairSharePreemptionThreshold(0).build()) + // Queue with minshare preemption enabled + .addQueue(new AllocationFileQueue.Builder("minshare") + .fairSharePreemptionThreshold(0) + .minSharePreemptionTimeout(0) + .minResources("2048mb,2vcores") + .build()) + // FAIR queue with fairshare preemption enabled + .addQueue(new AllocationFileQueue.Builder("fairshare") + .fairSharePreemptionThreshold(1) + .fairSharePreemptionTimeout(0) + .schedulingPolicy("fair") + .subQueue(new AllocationFileQueue.Builder("child") + .fairSharePreemptionThreshold(1) + .fairSharePreemptionTimeout(0) + .schedulingPolicy("fair").build()) + .build()) + // DRF queue with fairshare preemption enabled + .addQueue(new AllocationFileQueue.Builder("drf") + .fairSharePreemptionThreshold(1) + .fairSharePreemptionTimeout(0) + .schedulingPolicy("drf") + .subQueue(new AllocationFileQueue.Builder("child") + .fairSharePreemptionThreshold(1) + .fairSharePreemptionTimeout(0) + .schedulingPolicy("drf").build()) + .build()) + .writeToFile(ALLOC_FILE.getAbsolutePath()); assertTrue("Allocation file does not exist, not running the test", ALLOC_FILE.exists()); @@ -258,17 +249,6 @@ private void setupStarvedCluster() throws IOException { assertEquals(8, scheduler.getSchedulerApp(app).getLiveContainers().size()); } - private void addChildQueue(PrintWriter out, String policy) { - // Child queue under fairshare with same settings - out.println(""); - out.println("1" + - ""); - out.println("0" + - ""); - out.println("" + policy + ""); - out.println(""); - } - private void submitAppsToEachLeafQueue() { for (String queue : QUEUES) { createSchedulingRequest(1024, 1, "root." + queue, "user", 1); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFSLeafQueue.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFSLeafQueue.java index 9c47b6be0f9..6ecf80996d8 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFSLeafQueue.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFSLeafQueue.java @@ -24,9 +24,7 @@ import static org.mockito.Mockito.mock; import java.io.File; -import java.io.FileWriter; import java.io.IOException; -import java.io.PrintWriter; import java.util.ArrayList; import java.util.Collection; import java.util.Collections; @@ -49,6 +47,8 @@ import org.apache.hadoop.yarn.server.resourcemanager.scheduler.event.NodeAddedSchedulerEvent; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.event.NodeUpdateSchedulerEvent; import org.apache.hadoop.yarn.util.resource.ResourceUtils; +import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.allocationfile.AllocationFileQueue; +import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.allocationfile.AllocationFileWriter; import org.apache.hadoop.yarn.util.resource.Resources; import org.junit.After; import org.junit.Before; @@ -110,17 +110,14 @@ public void testUpdateDemand() { } @Test (timeout = 5000) - public void test() throws Exception { + public void test() { conf.set(FairSchedulerConfiguration.ALLOCATION_FILE, ALLOC_FILE); - PrintWriter out = new PrintWriter(new FileWriter(ALLOC_FILE)); - out.println(""); - out.println(""); - out.println("" + MAX_AM_SHARE + - ""); - out.println(""); - out.println(""); - out.println(""); - out.close(); + + AllocationFileWriter.create() + .queueMaxAMShareDefault(MAX_AM_SHARE) + .addQueue(new AllocationFileQueue.Builder("queueA").build()) + .addQueue(new AllocationFileQueue.Builder("queueB").build()) + .writeToFile(ALLOC_FILE); resourceManager = new MockRM(conf); resourceManager.start(); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFairScheduler.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFairScheduler.java index d9f29f1be11..21eccc2ef2d 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFairScheduler.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFairScheduler.java @@ -88,6 +88,16 @@ import org.apache.hadoop.yarn.server.resourcemanager.scheduler.event.NodeAddedSchedulerEvent; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.event.NodeRemovedSchedulerEvent; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.event.NodeUpdateSchedulerEvent; +import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair + .allocationfile.AllocationFileQueue; +import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair + .allocationfile.AllocationFileQueuePlacementPolicy; +import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair + .allocationfile.AllocationFileQueuePlacementRule; +import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair + .allocationfile.AllocationFileWriter; +import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair + .allocationfile.UserSettings; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.policies.DominantResourceFairnessPolicy; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.policies.FifoPolicy; import org.apache.hadoop.yarn.server.utils.BuilderUtils; @@ -99,9 +109,7 @@ import org.junit.Before; import org.junit.Test; import org.mockito.Mockito; -import org.xml.sax.SAXException; -import javax.xml.parsers.ParserConfigurationException; import java.io.File; import java.io.FileWriter; import java.io.IOException; @@ -338,27 +346,18 @@ public void testQueueMaximumCapacityAllocations() throws IOException { int tooHighQueueAllocation = RM_SCHEDULER_MAXIMUM_ALLOCATION_MB_VALUE +1; - PrintWriter out = new PrintWriter(new FileWriter(ALLOC_FILE)); - out.println(""); - out.println(""); - out.println(" "); - out.println( - " 512 mb 1 vcores"); - out.println(" "); - out.println(" "); - out.println(" "); - out.println(" "); - out.println( - " 2048 mb 3 vcores"); - out.println(" "); - out.println(" "); - out.println(" "); - out.println(" "); - out.println(" " + tooHighQueueAllocation - + " mb 1 vcores"); - out.println(" "); - out.println(""); - out.close(); + AllocationFileWriter.create() + .addQueue(new AllocationFileQueue.Builder("queueA") + .maxContainerAllocation("512 mb 1 vcores").build()) + .addQueue(new AllocationFileQueue.Builder("queueB").build()) + .addQueue(new AllocationFileQueue.Builder("queueC") + .maxContainerAllocation("2048 mb 3 vcores") + .subQueue(new AllocationFileQueue.Builder("queueD").build()) + .build()) + .addQueue(new AllocationFileQueue.Builder("queueE") + .maxContainerAllocation(tooHighQueueAllocation + " mb 1 vcores") + .build()) + .writeToFile(ALLOC_FILE); scheduler.init(conf); @@ -394,17 +393,12 @@ public void testNormalizationUsingQueueMaximumAllocation() int queueMaxAllocation = 4096; conf.set(FairSchedulerConfiguration.ALLOCATION_FILE, ALLOC_FILE); - PrintWriter out = new PrintWriter(new FileWriter(ALLOC_FILE)); - out.println(""); - out.println(""); - out.println(" "); - out.println(" " + queueMaxAllocation - + " mb 1 vcores" + ""); - out.println(" "); - out.println(" "); - out.println(" "); - out.println(""); - out.close(); + AllocationFileWriter.create() + .addQueue(new AllocationFileQueue.Builder("queueA") + .maxContainerAllocation(queueMaxAllocation + " mb 1 vcores") + .build()) + .addQueue(new AllocationFileQueue.Builder("queueB").build()) + .writeToFile(ALLOC_FILE); scheduler.init(conf); scheduler.start(); @@ -452,19 +446,17 @@ public void testFairShareWithHighMaxResources() throws IOException { // set queueA and queueB maxResources, // the sum of queueA and queueB maxResources is more than // Integer.MAX_VALUE. - PrintWriter out = new PrintWriter(new FileWriter(ALLOC_FILE)); - out.println(""); - out.println(""); - out.println(""); - out.println("1073741824 mb 1000 vcores"); - out.println(".25"); - out.println(""); - out.println(""); - out.println("1073741824 mb 1000 vcores"); - out.println(".75"); - out.println(""); - out.println(""); - out.close(); + + AllocationFileWriter.create() + .addQueue(new AllocationFileQueue.Builder("queueA") + .maxResources("1073741824 mb 1000 vcores") + .weight(.25f) + .build()) + .addQueue(new AllocationFileQueue.Builder("queueB") + .maxResources("1073741824 mb 1000 vcores") + .weight(.75f) + .build()) + .writeToFile(ALLOC_FILE); scheduler.init(conf); scheduler.start(); @@ -506,18 +498,16 @@ public void testFairShareWithHighMaxResources() throws IOException { public void testFairShareWithLowMaxResources() throws IOException { PrintWriter out = new PrintWriter(new FileWriter(ALLOC_FILE)); - out.println(""); - out.println(""); - out.println(" "); - out.println(" 1024 mb 1 vcores"); - out.println(" 0.75"); - out.println(" "); - out.println(" "); - out.println(" 3072 mb 3 vcores"); - out.println(" 0.25"); - out.println(" "); - out.println(""); - out.close(); + AllocationFileWriter.create() + .addQueue(new AllocationFileQueue.Builder("queueA") + .maxResources("1024 mb 1 vcores") + .weight(.75f) + .build()) + .addQueue(new AllocationFileQueue.Builder("queueB") + .maxResources("3072 mb 3 vcores") + .weight(.25f) + .build()) + .writeToFile(ALLOC_FILE); conf.set(FairSchedulerConfiguration.ALLOCATION_FILE, ALLOC_FILE); scheduler.init(conf); @@ -576,15 +566,12 @@ public void testFairShareWithLowMaxResources() throws IOException { */ @Test public void testChildMaxResources() throws IOException { - PrintWriter out = new PrintWriter(new FileWriter(ALLOC_FILE)); - - out.println(""); - out.println(""); - out.println(" "); - out.println(" 2048mb,2vcores"); - out.println(" "); - out.println(""); - out.close(); + AllocationFileWriter.create() + .addQueue(new AllocationFileQueue.Builder("queueA") + .parent(true) + .maxChildResources("2048mb,2vcores") + .build()) + .writeToFile(ALLOC_FILE); conf.set(FairSchedulerConfiguration.ALLOCATION_FILE, ALLOC_FILE); scheduler.init(conf); @@ -633,14 +620,12 @@ public void testChildMaxResources() throws IOException { assertEquals("App 2 is not running with the correct number of containers", 2, scheduler.getSchedulerApp(attId2).getLiveContainers().size()); - out = new PrintWriter(new FileWriter(ALLOC_FILE)); - out.println(""); - out.println(""); - out.println(" "); - out.println(" 3072mb,3vcores"); - out.println(" "); - out.println(""); - out.close(); + AllocationFileWriter.create() + .addQueue(new AllocationFileQueue.Builder("queueA") + .parent(true) + .maxChildResources("3072mb,3vcores") + .build()) + .writeToFile(ALLOC_FILE); scheduler.reinitialize(conf, resourceManager.getRMContext()); scheduler.update(); @@ -659,14 +644,12 @@ public void testChildMaxResources() throws IOException { assertEquals("App 2 is not running with the correct number of containers", 3, scheduler.getSchedulerApp(attId2).getLiveContainers().size()); - out = new PrintWriter(new FileWriter(ALLOC_FILE)); - out.println(""); - out.println(""); - out.println(" "); - out.println(" 1024mb,1vcores"); - out.println(" "); - out.println(""); - out.close(); + AllocationFileWriter.create() + .addQueue(new AllocationFileQueue.Builder("queueA") + .parent(true) + .maxChildResources("1024mb,1vcores") + .build()) + .writeToFile(ALLOC_FILE); //ensure that a 7th node heartbeat does not allocate more containers scheduler.handle(nodeEvent); @@ -696,17 +679,13 @@ private void drainEventsOnRM() { public void testFairShareWithZeroWeight() throws IOException { conf.set(FairSchedulerConfiguration.ALLOCATION_FILE, ALLOC_FILE); // set queueA and queueB weight zero. - PrintWriter out = new PrintWriter(new FileWriter(ALLOC_FILE)); - out.println(""); - out.println(""); - out.println(""); - out.println("0.0"); - out.println(""); - out.println(""); - out.println("0.0"); - out.println(""); - out.println(""); - out.close(); + + AllocationFileWriter.create() + .addQueue(new AllocationFileQueue.Builder("queueA") + .weight(0.0f).build()) + .addQueue(new AllocationFileQueue.Builder("queueB") + .weight(0.0f).build()) + .writeToFile(ALLOC_FILE); scheduler.init(conf); scheduler.start(); @@ -744,26 +723,23 @@ public void testFairShareWithZeroWeight() throws IOException { @Test public void testComputeMaxAMResource() throws IOException { conf.set(FairSchedulerConfiguration.ALLOCATION_FILE, ALLOC_FILE); - PrintWriter out = new PrintWriter(new FileWriter(ALLOC_FILE)); - out.println(""); - out.println(""); - out.println(""); - out.println("0"); - out.println("0.5"); - out.println("4096 mb 4 vcores"); - out.println(""); - out.println(""); - out.println("0.0"); - out.println("0.5"); - out.println(""); - out.println(""); - out.println("1"); - out.println("0.5"); - out.println(""); - out.println("drf" + - ""); - out.println(""); - out.close(); + + AllocationFileWriter.create() + .addQueue(new AllocationFileQueue.Builder("queueFSZeroWithMax") + .weight(0) + .maxAMShare(0.5) + .maxResources("4096 mb 4 vcores") + .build()) + .addQueue(new AllocationFileQueue.Builder("queueFSZeroWithAVL") + .weight(0.0f) + .maxAMShare(0.5) + .build()) + .addQueue(new AllocationFileQueue.Builder("queueFSNonZero") + .weight(1) + .maxAMShare(0.5) + .build()) + .defaultQueueSchedulingPolicy("drf") + .writeToFile(ALLOC_FILE); scheduler.init(conf); scheduler.start(); @@ -878,19 +854,16 @@ public void testFairShareWithZeroWeightNoneZeroMinRes() throws IOException { conf.set(FairSchedulerConfiguration.ALLOCATION_FILE, ALLOC_FILE); // set queueA and queueB weight zero. // set queueA and queueB minResources 1. - PrintWriter out = new PrintWriter(new FileWriter(ALLOC_FILE)); - out.println(""); - out.println(""); - out.println(""); - out.println("1 mb 1 vcores"); - out.println("0.0"); - out.println(""); - out.println(""); - out.println("1 mb 1 vcores"); - out.println("0.0"); - out.println(""); - out.println(""); - out.close(); + AllocationFileWriter.create() + .addQueue(new AllocationFileQueue.Builder("queueA") + .weight(0) + .minResources("1 mb 1 vcores") + .build()) + .addQueue(new AllocationFileQueue.Builder("queueB") + .minResources("1 mb 1 vcores") + .weight(0.0f) + .build()) + .writeToFile(ALLOC_FILE); scheduler.init(conf); scheduler.start(); @@ -928,19 +901,16 @@ public void testFairShareWithNoneZeroWeightNoneZeroMinRes() conf.set(FairSchedulerConfiguration.ALLOCATION_FILE, ALLOC_FILE); // set queueA and queueB weight 0.5. // set queueA and queueB minResources 1024. - PrintWriter out = new PrintWriter(new FileWriter(ALLOC_FILE)); - out.println(""); - out.println(""); - out.println(""); - out.println("1024 mb 1 vcores"); - out.println("0.5"); - out.println(""); - out.println(""); - out.println("1024 mb 1 vcores"); - out.println("0.5"); - out.println(""); - out.println(""); - out.close(); + AllocationFileWriter.create() + .addQueue(new AllocationFileQueue.Builder("queueA") + .weight(0.5f) + .minResources("1024 mb 1 vcores") + .build()) + .addQueue(new AllocationFileQueue.Builder("queueB") + .weight(0.5f) + .minResources("1024 mb 1 vcores") + .build()) + .writeToFile(ALLOC_FILE); scheduler.init(conf); scheduler.start(); @@ -976,17 +946,14 @@ public void testFairShareWithNoneZeroWeightNoneZeroMinRes() public void testQueueInfo() throws IOException { conf.set(FairSchedulerConfiguration.ALLOCATION_FILE, ALLOC_FILE); - PrintWriter out = new PrintWriter(new FileWriter(ALLOC_FILE)); - out.println(""); - out.println(""); - out.println(""); - out.println(".25"); - out.println(""); - out.println(""); - out.println(".75"); - out.println(""); - out.println(""); - out.close(); + AllocationFileWriter.create() + .addQueue(new AllocationFileQueue.Builder("queueA") + .weight(0.25f) + .build()) + .addQueue(new AllocationFileQueue.Builder("queueB") + .weight(0.75f) + .build()) + .writeToFile(ALLOC_FILE); scheduler.init(conf); scheduler.start(); @@ -1502,19 +1469,15 @@ public void testReservationThresholdWithAssignMultiple() throws Exception { public void testContainerReservationAttemptExceedingQueueMax() throws Exception { conf.set(FairSchedulerConfiguration.ALLOCATION_FILE, ALLOC_FILE); - PrintWriter out = new PrintWriter(new FileWriter(ALLOC_FILE)); - out.println(""); - out.println(""); - out.println(""); - out.println(""); - out.println("2048mb,5vcores"); - out.println(""); - out.println(""); - out.println("2048mb,10vcores"); - out.println(""); - out.println(""); - out.println(""); - out.close(); + + AllocationFileWriter.create() + .addQueue(new AllocationFileQueue.Builder("root") + .subQueue(new AllocationFileQueue.Builder("queue1") + .maxResources("2048mb,5vcores").build()) + .subQueue(new AllocationFileQueue.Builder("queue2") + .maxResources("2048mb,10vcores").build()) + .build()) + .writeToFile(ALLOC_FILE); scheduler.init(conf); scheduler.start(); @@ -1564,22 +1527,20 @@ public void testContainerReservationAttemptExceedingQueueMax() public void testRequestAMResourceInZeroFairShareQueue() throws Exception { conf.set(FairSchedulerConfiguration.ALLOCATION_FILE, ALLOC_FILE); - PrintWriter out = new PrintWriter(new FileWriter(ALLOC_FILE)); - out.println(""); - out.println(""); - out.println(""); - out.println("0.0"); - out.println("4096mb,10vcores"); - out.println("0.5"); - out.println(""); - out.println(""); - out.println("2.0"); - out.println(""); - out.println(""); - out.println("0.000001"); - out.println(""); - out.println(""); - out.close(); + AllocationFileWriter.create() + .addQueue(new AllocationFileQueue.Builder("queue1") + .weight(0) + .maxAMShare(0.5) + .maxResources("4096mb,10vcores") + .build()) + .addQueue(new AllocationFileQueue.Builder("queue2") + .weight(2.0f) + .build()) + .addQueue(new AllocationFileQueue.Builder("queue3") + .weight(0.000001f) + .build()) + .defaultQueueSchedulingPolicy("drf") + .writeToFile(ALLOC_FILE); scheduler.init(conf); scheduler.start(); @@ -1637,19 +1598,15 @@ public void testRequestAMResourceInZeroFairShareQueue() throws Exception { @Test (timeout = 500000) public void testContainerReservationNotExceedingQueueMax() throws Exception { conf.set(FairSchedulerConfiguration.ALLOCATION_FILE, ALLOC_FILE); - PrintWriter out = new PrintWriter(new FileWriter(ALLOC_FILE)); - out.println(""); - out.println(""); - out.println(""); - out.println(""); - out.println("3072mb,10vcores"); - out.println(""); - out.println(""); - out.println("2048mb,10vcores"); - out.println(""); - out.println(""); - out.println(""); - out.close(); + + AllocationFileWriter.create() + .addQueue(new AllocationFileQueue.Builder("root") + .subQueue(new AllocationFileQueue.Builder("queue1") + .maxResources("3072mb,10vcores").build()) + .subQueue(new AllocationFileQueue.Builder("queue2") + .maxResources("2048mb,10vcores").build()) + .build()) + .writeToFile(ALLOC_FILE); scheduler.init(conf); scheduler.start(); @@ -1698,19 +1655,14 @@ public void testContainerReservationNotExceedingQueueMax() throws Exception { getCurrentReservation().getMemorySize()); // Now reduce max Resources of queue1 down to 2048 - out = new PrintWriter(new FileWriter(ALLOC_FILE)); - out.println(""); - out.println(""); - out.println(""); - out.println(""); - out.println("2048mb,10vcores"); - out.println(""); - out.println(""); - out.println("2048mb,10vcores"); - out.println(""); - out.println(""); - out.println(""); - out.close(); + AllocationFileWriter.create() + .addQueue(new AllocationFileQueue.Builder("root") + .subQueue(new AllocationFileQueue.Builder("queue1") + .maxResources("2048mb,10vcores").build()) + .subQueue(new AllocationFileQueue.Builder("queue2") + .maxResources("2048mb,10vcores").build()) + .build()) + .writeToFile(ALLOC_FILE); scheduler.reinitialize(conf, resourceManager.getRMContext()); @@ -1731,13 +1683,9 @@ public void testContainerReservationNotExceedingQueueMax() throws Exception { public void testReservationThresholdGatesReservations() throws Exception { conf.set(FairSchedulerConfiguration.ALLOCATION_FILE, ALLOC_FILE); - PrintWriter out = new PrintWriter(new FileWriter(ALLOC_FILE)); - out.println(""); - out.println(""); - out.println("drf" + - ""); - out.println(""); - out.close(); + AllocationFileWriter.create() + .defaultQueueSchedulingPolicy("drf") + .writeToFile(ALLOC_FILE); // Set threshold to 2 * 1024 ==> 2048 MB & 2 * 1 ==> 2 vcores (test will // use vcores) @@ -1883,17 +1831,13 @@ public void testQueueuNameWithPeriods() throws Exception { public void testFairShareWithMinAlloc() throws Exception { conf.set(FairSchedulerConfiguration.ALLOCATION_FILE, ALLOC_FILE); - PrintWriter out = new PrintWriter(new FileWriter(ALLOC_FILE)); - out.println(""); - out.println(""); - out.println(""); - out.println("1024mb,0vcores"); - out.println(""); - out.println(""); - out.println("2048mb,0vcores"); - out.println(""); - out.println(""); - out.close(); + AllocationFileWriter.create() + .addQueue(new AllocationFileQueue.Builder("queueA") + .minResources("1024mb,0vcores").build()) + .addQueue(new AllocationFileQueue.Builder("queueB") + .minResources("2048mb,0vcores") + .build()) + .writeToFile(ALLOC_FILE); scheduler.init(conf); scheduler.start(); @@ -1928,21 +1872,18 @@ else if (p.getName().equals("root.queueB")) { public void testFairShareAndWeightsInNestedUserQueueRule() throws Exception { conf.set(FairSchedulerConfiguration.ALLOCATION_FILE, ALLOC_FILE); - PrintWriter out = new PrintWriter(new FileWriter(ALLOC_FILE)); - - out.println(""); - out.println(""); - out.println(""); - out.println("1024mb,0vcores"); - out.println(""); - out.println(""); - out.println(""); - out.println(" "); - out.println(""); - out.println(""); - out.println(""); - out.println(""); - out.close(); + AllocationFileWriter.create() + .addQueue(new AllocationFileQueue.Builder("parentq") + .parent(true) + .minResources("1024mb,0vcores") + .build()) + .queuePlacementPolicy(new AllocationFileQueuePlacementPolicy() + .addRule(new AllocationFileQueuePlacementRule("nestedUserQueue") + .addNestedRule( + new AllocationFileQueuePlacementRule("specified") + .create(false))) + .addRule(new AllocationFileQueuePlacementRule("default"))) + .writeToFile(ALLOC_FILE); RMApp rmApp1 = new MockRMApp(0, 0, RMAppState.NEW); RMApp rmApp2 = new MockRMApp(1, 1, RMAppState.NEW); @@ -1989,21 +1930,16 @@ public void testFairShareAndWeightsInNestedUserQueueRule() throws Exception { public void testSteadyFairShareWithReloadAndNodeAddRemove() throws Exception { conf.set(FairSchedulerConfiguration.ALLOCATION_FILE, ALLOC_FILE); - PrintWriter out = new PrintWriter(new FileWriter(ALLOC_FILE)); - out.println(""); - out.println(""); - out.println("fair"); - out.println(""); - out.println(" drf"); - out.println(" "); - out.println(" 1"); - out.println(" "); - out.println(" "); - out.println(" 1"); - out.println(" "); - out.println(""); - out.println(""); - out.close(); + AllocationFileWriter.create() + .defaultQueueSchedulingPolicy("fair") + .addQueue(new AllocationFileQueue.Builder("root") + .schedulingPolicy("drf") + .subQueue(new AllocationFileQueue.Builder("child1") + .weight(1).build()) + .subQueue(new AllocationFileQueue.Builder("child2") + .weight(1).build()) + .build()) + .writeToFile(ALLOC_FILE); scheduler.init(conf); scheduler.start(); @@ -2031,24 +1967,19 @@ public void testSteadyFairShareWithReloadAndNodeAddRemove() throws Exception { .getSteadyFairShare().getMemorySize()); // Reload the allocation configuration file - out = new PrintWriter(new FileWriter(ALLOC_FILE)); - out.println(""); - out.println(""); - out.println("fair"); - out.println(""); - out.println(" drf"); - out.println(" "); - out.println(" 1"); - out.println(" "); - out.println(" "); - out.println(" 2"); - out.println(" "); - out.println(" "); - out.println(" 2"); - out.println(" "); - out.println(""); - out.println(""); - out.close(); + AllocationFileWriter.create() + .defaultQueueSchedulingPolicy("fair") + .addQueue(new AllocationFileQueue.Builder("root") + .schedulingPolicy("drf") + .subQueue(new AllocationFileQueue.Builder("child1") + .weight(1).build()) + .subQueue(new AllocationFileQueue.Builder("child2") + .weight(2).build()) + .subQueue(new AllocationFileQueue.Builder("child3") + .weight(2).build()) + .build()) + .writeToFile(ALLOC_FILE); + scheduler.reinitialize(conf, resourceManager.getRMContext()); // The steady fair shares for all queues should be updated @@ -2168,27 +2099,19 @@ public void testQueueDemandCalculation() throws Exception { } @Test - public void testHierarchicalQueueAllocationFileParsing() throws IOException, SAXException, - AllocationConfigurationException, ParserConfigurationException { + public void testHierarchicalQueueAllocationFileParsing() throws IOException { conf.set(FairSchedulerConfiguration.ALLOCATION_FILE, ALLOC_FILE); - - PrintWriter out = new PrintWriter(new FileWriter(ALLOC_FILE)); - out.println(""); - out.println(""); - out.println(""); - out.println("2048mb,0vcores"); - out.println(""); - out.println(""); - out.println("2048mb,0vcores"); - out.println(""); - out.println("2048mb,0vcores"); - out.println(""); - out.println(""); - out.println("2048mb,0vcores"); - out.println(""); - out.println(""); - out.println(""); - out.close(); + AllocationFileWriter.create() + .addQueue(new AllocationFileQueue.Builder("queueA") + .minResources("2048mb,0vcores").build()) + .addQueue(new AllocationFileQueue.Builder("queueB") + .minResources("2048mb,0vcores") + .subQueue(new AllocationFileQueue.Builder("queueC") + .minResources("2048mb,0vcores").build()) + .subQueue(new AllocationFileQueue.Builder("queueD") + .minResources("2048mb,0vcores").build()) + .build()) + .writeToFile(ALLOC_FILE); scheduler.init(conf); scheduler.start(); @@ -2209,27 +2132,22 @@ public void testHierarchicalQueueAllocationFileParsing() throws IOException, SAX public void testConfigureRootQueue() throws Exception { conf.set(FairSchedulerConfiguration.ALLOCATION_FILE, ALLOC_FILE); - PrintWriter out = new PrintWriter(new FileWriter(ALLOC_FILE)); - out.println(""); - out.println(""); - out.println("fair"); - out.println(""); - out.println(" drf"); - out.println(" "); - out.println(" 1024mb,1vcores"); - out.println(" "); - out.println(" "); - out.println(" 1024mb,4vcores"); - out.println(" "); - out.println(" 100"); - out.println(" 120"); - out.println(" .5"); - out.println(""); - out.println("300"); - out.println("200"); - out.println(".6"); - out.println(""); - out.close(); + AllocationFileWriter.create() + .defaultQueueSchedulingPolicy("fair") + .defaultFairSharePreemptionTimeout(300) + .defaultMinSharePreemptionTimeout(200) + .defaultFairSharePreemptionThreshold(.6) + .addQueue(new AllocationFileQueue.Builder("root") + .schedulingPolicy("drf") + .fairSharePreemptionTimeout(100) + .fairSharePreemptionThreshold(.5) + .minSharePreemptionTimeout(120) + .subQueue(new AllocationFileQueue.Builder("child1") + .minResources("1024mb,1vcores").build()) + .subQueue(new AllocationFileQueue.Builder("child2") + .minResources("1024mb,4vcores").build()) + .build()) + .writeToFile(ALLOC_FILE); scheduler.init(conf); scheduler.start(); @@ -2284,14 +2202,10 @@ public void testUserMaxRunningApps() throws Exception { // Set max running apps conf.set(FairSchedulerConfiguration.ALLOCATION_FILE, ALLOC_FILE); - PrintWriter out = new PrintWriter(new FileWriter(ALLOC_FILE)); - out.println(""); - out.println(""); - out.println(""); - out.println("1"); - out.println(""); - out.println(""); - out.close(); + AllocationFileWriter.create() + .userSettings(new UserSettings.Builder("user1") + .maxRunningApps(1).build()) + .writeToFile(ALLOC_FILE); scheduler.init(conf); scheduler.start(); @@ -2336,65 +2250,57 @@ public void testUserMaxRunningApps() throws Exception { @Test (timeout = 5000) public void testIncreaseQueueMaxRunningAppsOnTheFly() throws Exception { - String allocBefore = "" + - "" + - "" + - "" + - "1" + - "" + - "" + - ""; - - String allocAfter = "" + - "" + - "" + - "" + - "3" + - "" + - "" + - ""; + AllocationFileWriter allocBefore = AllocationFileWriter.create() + .addQueue(new AllocationFileQueue.Builder("root") + .subQueue( + new AllocationFileQueue.Builder("queue1") + .maxRunningApps(1) + .build()) + .build()); + + AllocationFileWriter allocAfter = AllocationFileWriter.create() + .addQueue(new AllocationFileQueue.Builder("root") + .subQueue( + new AllocationFileQueue.Builder("queue1") + .maxRunningApps(3) + .build()) + .build()); testIncreaseQueueSettingOnTheFlyInternal(allocBefore, allocAfter); } @Test (timeout = 5000) public void testIncreaseUserMaxRunningAppsOnTheFly() throws Exception { - String allocBefore = ""+ - ""+ - ""+ - ""+ - "10"+ - ""+ - ""+ - ""+ - "1"+ - ""+ - ""; - - String allocAfter = ""+ - ""+ - ""+ - ""+ - "10"+ - ""+ - ""+ - ""+ - "3"+ - ""+ - ""; + AllocationFileWriter allocBefore = AllocationFileWriter.create() + .addQueue(new AllocationFileQueue.Builder("root") + .subQueue( + new AllocationFileQueue.Builder("queue1") + .maxRunningApps(10) + .build()) + .build()) + .userSettings(new UserSettings.Builder("user1") + .maxRunningApps(1).build()); + + AllocationFileWriter allocAfter = AllocationFileWriter.create() + .addQueue(new AllocationFileQueue.Builder("root") + .subQueue( + new AllocationFileQueue.Builder("queue1") + .maxRunningApps(10) + .build()) + .build()) + .userSettings(new UserSettings.Builder("user1") + .maxRunningApps(3).build()); testIncreaseQueueSettingOnTheFlyInternal(allocBefore, allocAfter); } - private void testIncreaseQueueSettingOnTheFlyInternal(String allocBefore, - String allocAfter) throws Exception { + private void testIncreaseQueueSettingOnTheFlyInternal( + AllocationFileWriter allocBefore, + AllocationFileWriter allocAfter) throws Exception { // Set max running apps conf.set(FairSchedulerConfiguration.ALLOCATION_FILE, ALLOC_FILE); - PrintWriter out = new PrintWriter(new FileWriter(ALLOC_FILE)); - out.println(allocBefore); - out.close(); - + allocBefore.writeToFile(ALLOC_FILE); scheduler.init(conf); scheduler.start(); scheduler.reinitialize(conf, resourceManager.getRMContext()); @@ -2442,9 +2348,7 @@ private void testIncreaseQueueSettingOnTheFlyInternal(String allocBefore, // App 4 should not be running assertEquals(0, scheduler.getSchedulerApp(attId4).getLiveContainers().size()); - out = new PrintWriter(new FileWriter(ALLOC_FILE)); - out.println(allocAfter); - out.close(); + allocAfter.writeToFile(ALLOC_FILE); scheduler.reinitialize(conf, resourceManager.getRMContext()); scheduler.update(); @@ -2479,64 +2383,56 @@ private void testIncreaseQueueSettingOnTheFlyInternal(String allocBefore, @Test (timeout = 5000) public void testDecreaseQueueMaxRunningAppsOnTheFly() throws Exception { - String allocBefore = "" + - "" + - "" + - "" + - "3" + - "" + - "" + - ""; - - String allocAfter = "" + - "" + - "" + - "" + - "1" + - "" + - "" + - ""; + AllocationFileWriter allocBefore = AllocationFileWriter.create() + .addQueue(new AllocationFileQueue.Builder("root") + .subQueue( + new AllocationFileQueue.Builder("queue1") + .maxRunningApps(3) + .build()) + .build()); + + AllocationFileWriter allocAfter = AllocationFileWriter.create() + .addQueue(new AllocationFileQueue.Builder("root") + .subQueue( + new AllocationFileQueue.Builder("queue1") + .maxRunningApps(1) + .build()) + .build()); testDecreaseQueueSettingOnTheFlyInternal(allocBefore, allocAfter); } @Test (timeout = 5000) public void testDecreaseUserMaxRunningAppsOnTheFly() throws Exception { - String allocBefore = ""+ - ""+ - ""+ - ""+ - "10"+ - ""+ - ""+ - ""+ - "3"+ - ""+ - ""; - - String allocAfter = ""+ - ""+ - ""+ - ""+ - "10"+ - ""+ - ""+ - ""+ - "1"+ - ""+ - ""; + AllocationFileWriter allocBefore = AllocationFileWriter.create() + .addQueue(new AllocationFileQueue.Builder("root") + .subQueue( + new AllocationFileQueue.Builder("queue1") + .maxRunningApps(10) + .build()) + .build()) + .userSettings(new UserSettings.Builder("user1") + .maxRunningApps(3).build()); + + AllocationFileWriter allocAfter = AllocationFileWriter.create() + .addQueue(new AllocationFileQueue.Builder("root") + .subQueue( + new AllocationFileQueue.Builder("queue1") + .maxRunningApps(10) + .build()) + .build()) + .userSettings(new UserSettings.Builder("user1") + .maxRunningApps(1).build()); testDecreaseQueueSettingOnTheFlyInternal(allocBefore, allocAfter); } - private void testDecreaseQueueSettingOnTheFlyInternal(String allocBefore, - String allocAfter) throws Exception { + private void testDecreaseQueueSettingOnTheFlyInternal( + AllocationFileWriter allocBefore, + AllocationFileWriter allocAfter) throws Exception { // Set max running apps conf.set(FairSchedulerConfiguration.ALLOCATION_FILE, ALLOC_FILE); - - PrintWriter out = new PrintWriter(new FileWriter(ALLOC_FILE)); - out.println(allocBefore); - out.close(); + allocBefore.writeToFile(ALLOC_FILE); scheduler.init(conf); scheduler.start(); @@ -2585,9 +2481,7 @@ private void testDecreaseQueueSettingOnTheFlyInternal(String allocBefore, // App 4 should not be running assertEquals(0, scheduler.getSchedulerApp(attId4).getLiveContainers().size()); - out = new PrintWriter(new FileWriter(ALLOC_FILE)); - out.println(allocAfter); - out.close(); + allocAfter.writeToFile(ALLOC_FILE); scheduler.reinitialize(conf, resourceManager.getRMContext()); scheduler.update(); @@ -2700,19 +2594,16 @@ public void testAclSubmitApplication() throws Exception { // Set acl's conf.set(FairSchedulerConfiguration.ALLOCATION_FILE, ALLOC_FILE); - PrintWriter out = new PrintWriter(new FileWriter(ALLOC_FILE)); - out.println(""); - out.println(""); - out.println(""); - out.println(" "); - out.println(" "); - out.println(" "); - out.println(" norealuserhasthisname"); - out.println(" norealuserhasthisname"); - out.println(" "); - out.println(""); - out.println(""); - out.close(); + AllocationFileWriter.create() + .addQueue(new AllocationFileQueue.Builder("root") + .aclSubmitApps(" ") + .aclAdministerApps(" ") + .subQueue(new AllocationFileQueue.Builder("queue1") + .aclSubmitApps("norealuserhasthisname") + .aclAdministerApps("norealuserhasthisname") + .build()) + .build()) + .writeToFile(ALLOC_FILE); scheduler.init(conf); scheduler.start(); @@ -3039,19 +2930,17 @@ public void testAssignContainer() throws Exception { public void testNotAllowSubmitApplication() throws Exception { // Set acl's conf.set(FairSchedulerConfiguration.ALLOCATION_FILE, ALLOC_FILE); - PrintWriter out = new PrintWriter(new FileWriter(ALLOC_FILE)); - out.println(""); - out.println(""); - out.println(""); - out.println(" "); - out.println(" "); - out.println(" "); - out.println(" userallow"); - out.println(" userallow"); - out.println(" "); - out.println(""); - out.println(""); - out.close(); + + AllocationFileWriter.create() + .addQueue(new AllocationFileQueue.Builder("root") + .aclSubmitApps(" ") + .aclAdministerApps(" ") + .subQueue(new AllocationFileQueue.Builder("queue1") + .aclSubmitApps("userallow") + .aclAdministerApps("userallow") + .build()) + .build()) + .writeToFile(ALLOC_FILE); scheduler.init(conf); scheduler.start(); @@ -3593,17 +3482,12 @@ private void verifyQueueNumRunnable(String queueName, int numRunnableInQueue, public void testUserAndQueueMaxRunningApps() throws Exception { conf.set(FairSchedulerConfiguration.ALLOCATION_FILE, ALLOC_FILE); - PrintWriter out = new PrintWriter(new FileWriter(ALLOC_FILE)); - out.println(""); - out.println(""); - out.println(""); - out.println("2"); - out.println(""); - out.println(""); - out.println("1"); - out.println(""); - out.println(""); - out.close(); + AllocationFileWriter.create() + .addQueue(new AllocationFileQueue.Builder("queue1") + .maxRunningApps(2).build()) + .userSettings(new UserSettings.Builder("user1") + .maxRunningApps(1).build()) + .writeToFile(ALLOC_FILE); scheduler.init(conf); scheduler.start(); @@ -3646,14 +3530,10 @@ public void testMultipleCompletedEvent() throws Exception { // Set up a fair scheduler conf.set(FairSchedulerConfiguration.ALLOCATION_FILE, ALLOC_FILE); - PrintWriter out = new PrintWriter(new FileWriter(ALLOC_FILE)); - out.println(""); - out.println(""); - out.println(""); - out.println("0.2"); - out.println(""); - out.println(""); - out.close(); + AllocationFileWriter.create() + .addQueue(new AllocationFileQueue.Builder("queue1") + .maxAMShare(0.2).build()) + .writeToFile(ALLOC_FILE); scheduler.init(conf); scheduler.start(); @@ -3696,14 +3576,10 @@ public void testMultipleCompletedEvent() throws Exception { public void testQueueMaxAMShare() throws Exception { conf.set(FairSchedulerConfiguration.ALLOCATION_FILE, ALLOC_FILE); - PrintWriter out = new PrintWriter(new FileWriter(ALLOC_FILE)); - out.println(""); - out.println(""); - out.println(""); - out.println("0.2"); - out.println(""); - out.println(""); - out.close(); + AllocationFileWriter.create() + .addQueue(new AllocationFileQueue.Builder("queue1") + .maxAMShare(0.2).build()) + .writeToFile(ALLOC_FILE); scheduler.init(conf); scheduler.start(); @@ -3926,25 +3802,18 @@ public void testQueueMaxAMShareDefault() throws Exception { conf.set(FairSchedulerConfiguration.ALLOCATION_FILE, ALLOC_FILE); conf.setInt(YarnConfiguration.RM_SCHEDULER_MAXIMUM_ALLOCATION_VCORES, 6); - PrintWriter out = new PrintWriter(new FileWriter(ALLOC_FILE)); - out.println(""); - out.println(""); - out.println(""); - out.println(""); - out.println(""); - out.println("0.4"); - out.println(""); - out.println(""); - out.println("10240 mb 4 vcores"); - out.println(""); - out.println(""); - out.println(""); - out.println(""); - out.println(""); - out.println( - "fair"); - out.println(""); - out.close(); + AllocationFileWriter.create() + .defaultQueueSchedulingPolicy("fair") + .addQueue(new AllocationFileQueue.Builder("queue1").build()) + .addQueue(new AllocationFileQueue.Builder("queue2") + .maxAMShare(0.4f) + .build()) + .addQueue(new AllocationFileQueue.Builder("queue3") + .maxResources("10240 mb 4 vcores") + .build()) + .addQueue(new AllocationFileQueue.Builder("queue4").build()) + .addQueue(new AllocationFileQueue.Builder("queue5").build()) + .writeToFile(ALLOC_FILE); scheduler.init(conf); scheduler.start(); @@ -4087,14 +3956,10 @@ public void testQueueMaxAMShareDefault() throws Exception { public void testQueueMaxAMShareWithContainerReservation() throws Exception { conf.set(FairSchedulerConfiguration.ALLOCATION_FILE, ALLOC_FILE); conf.setFloat(FairSchedulerConfiguration.RESERVABLE_NODES, 1f); - PrintWriter out = new PrintWriter(new FileWriter(ALLOC_FILE)); - out.println(""); - out.println(""); - out.println(""); - out.println("0.5"); - out.println(""); - out.println(""); - out.close(); + AllocationFileWriter.create() + .addQueue(new AllocationFileQueue.Builder("queue1") + .maxAMShare(0.5).build()) + .writeToFile(ALLOC_FILE); scheduler.init(conf); scheduler.start(); @@ -4353,19 +4218,16 @@ public void testMaxRunningAppsHierarchicalQueues() throws Exception { ControlledClock clock = new ControlledClock(); scheduler.setClock(clock); - PrintWriter out = new PrintWriter(new FileWriter(ALLOC_FILE)); - out.println(""); - out.println(""); - out.println(""); - out.println(" 3"); - out.println(" "); - out.println(" "); - out.println(" "); - out.println(" 1"); - out.println(" "); - out.println(""); - out.println(""); - out.close(); + AllocationFileWriter.create() + .addQueue(new AllocationFileQueue.Builder("queue1") + .maxRunningApps(3) + .subQueue(new AllocationFileQueue.Builder("sub1").build()) + .subQueue(new AllocationFileQueue.Builder("sub2").build()) + .subQueue(new AllocationFileQueue.Builder("sub3") + .maxRunningApps(1) + .build()) + .build()) + .writeToFile(ALLOC_FILE); scheduler.init(conf); scheduler.start(); @@ -4479,11 +4341,8 @@ public void testDefaultRuleInitializesProperlyWhenPolicyNotConfigured() conf.setBoolean(FairSchedulerConfiguration.ALLOW_UNDECLARED_POOLS, false); // Create an alloc file with no queue placement policy - PrintWriter out = new PrintWriter(new FileWriter(ALLOC_FILE)); - out.println(""); - out.println(""); - out.println(""); - out.close(); + AllocationFileWriter.create() + .writeToFile(ALLOC_FILE); scheduler.init(conf); scheduler.start(); @@ -4944,16 +4803,13 @@ public void testQueueNameWithTrailingSpace() throws Exception { } @Test - public void testEmptyQueueNameInConfigFile() throws IOException { + public void testEmptyQueueNameInConfigFile() { conf.set(FairSchedulerConfiguration.ALLOCATION_FILE, ALLOC_FILE); // set empty queue name - PrintWriter out = new PrintWriter(new FileWriter(ALLOC_FILE)); - out.println(""); - out.println(""); - out.println(""); - out.println(""); - out.println(""); - out.close(); + AllocationFileWriter.create() + .addQueue(new AllocationFileQueue.Builder("").build()) + .writeToFile(ALLOC_FILE); + try { scheduler.init(conf); Assert.fail("scheduler init should fail because" + @@ -5188,6 +5044,8 @@ public void testRefreshQueuesWhenRMHA() throws Exception { new HAServiceProtocol.StateChangeRequestInfo( HAServiceProtocol.RequestSource.REQUEST_BY_USER); + //ensure ALLOC_FILE contains an 'empty' config + AllocationFileWriter.create().writeToFile(ALLOC_FILE); // 1. start a standby RM, file 'ALLOC_FILE' is empty, so there is no queues MockRM rm1 = new MockRM(conf, null); rm1.init(conf); @@ -5195,14 +5053,10 @@ public void testRefreshQueuesWhenRMHA() throws Exception { rm1.getAdminService().transitionToStandby(requestInfo); // 2. add a new queue "test_queue" - PrintWriter out = new PrintWriter(new FileWriter(ALLOC_FILE)); - out.println(""); - out.println(""); - out.println(""); - out.println(" 3"); - out.println(""); - out.println(""); - out.close(); + AllocationFileWriter.create() + .addQueue(new AllocationFileQueue.Builder("test_queue") + .maxRunningApps(3).build()) + .writeToFile(ALLOC_FILE); conf.set(YarnConfiguration.RM_STORE, MemoryRMStateStore.class.getName()); // 3. start a active RM @@ -5338,16 +5192,12 @@ public void testUpdateDemand() throws IOException { public void testDumpState() throws IOException { conf.set(FairSchedulerConfiguration.ALLOCATION_FILE, ALLOC_FILE); - PrintWriter out = new PrintWriter(new FileWriter(ALLOC_FILE)); - out.println(""); - out.println(""); - out.println(""); - out.println(" "); - out.println(" 1"); - out.println(" "); - out.println(""); - out.println(""); - out.close(); + AllocationFileWriter.create() + .addQueue(new AllocationFileQueue.Builder("parent") + .subQueue(new AllocationFileQueue.Builder("child1") + .weight(1).build()) + .build()) + .writeToFile(ALLOC_FILE); ControlledClock clock = new ControlledClock(); scheduler.setClock(clock); @@ -5528,29 +5378,25 @@ private void testAppRejectedToQueueWithZeroCapacityOfResource(String resource) + "maximum queue resources: .+")); } - private void generateAllocationFileWithZeroResource(String resource) - throws IOException { - PrintWriter out = new PrintWriter(new FileWriter(ALLOC_FILE)); - out.println(""); - out.println(""); - out.println(""); - + private void generateAllocationFileWithZeroResource(String resource) { String resources = ""; if (resource.equals(ResourceInformation.MEMORY_URI)) { resources = "0 mb,2vcores"; } else if (resource.equals(ResourceInformation.VCORES_URI)) { resources = "10000 mb,0vcores"; } - out.println("" + resources + ""); - out.println("" + resources + ""); - out.println("2.0"); - out.println(""); - out.println(""); - out.println("1 mb 1 vcores"); - out.println("0.0"); - out.println(""); - out.println(""); - out.close(); + + AllocationFileWriter.create() + .addQueue(new AllocationFileQueue.Builder("queueA") + .minResources(resources) + .maxResources(resources) + .weight(2.0f) + .build()) + .addQueue(new AllocationFileQueue.Builder("queueB") + .minResources("1 mb 1 vcores") + .weight(0.0f) + .build()) + .writeToFile(ALLOC_FILE); } @Test diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFairSchedulerFairShare.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFairSchedulerFairShare.java index a79aacc196e..cf6a36abdb4 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFairSchedulerFairShare.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFairSchedulerFairShare.java @@ -21,9 +21,7 @@ import static org.junit.Assert.assertEquals; import java.io.File; -import java.io.FileWriter; import java.io.IOException; -import java.io.PrintWriter; import java.util.Collection; import org.apache.hadoop.yarn.api.records.ApplicationAttemptId; @@ -33,6 +31,11 @@ import org.apache.hadoop.yarn.server.resourcemanager.rmnode.RMNode; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.event.AppAttemptRemovedSchedulerEvent; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.event.NodeAddedSchedulerEvent; + +import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair + .allocationfile.AllocationFileQueue; +import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair + .allocationfile.AllocationFileWriter; import org.apache.hadoop.yarn.util.resource.Resources; import org.junit.After; import org.junit.Before; @@ -57,34 +60,29 @@ public void teardown() { conf = null; } - private void createClusterWithQueuesAndOneNode(int mem, String policy) - throws IOException { + private void createClusterWithQueuesAndOneNode(int mem, String policy) { createClusterWithQueuesAndOneNode(mem, 0, policy); } private void createClusterWithQueuesAndOneNode(int mem, int vCores, - String policy) throws IOException { - PrintWriter out = new PrintWriter(new FileWriter(ALLOC_FILE)); - out.println(""); - out.println(""); - out.println(""); - out.println(" "); - out.println(" 8"); - out.println(" "); - out.println(" "); - out.println(" "); - out.println(" "); - out.println(" "); - out.println(" "); - out.println(" 1"); - out.println(" "); - out.println(" "); - out.println(" "); - out.println(""); - out.println("" + policy - + ""); - out.println(""); - out.close(); + String policy) { + AllocationFileWriter.create() + .defaultQueueSchedulingPolicy(policy) + .addQueue(new AllocationFileQueue.Builder("root") + .subQueue(new AllocationFileQueue.Builder("parentA") + .weight(8) + .subQueue(new AllocationFileQueue.Builder("childA1").build()) + .subQueue(new AllocationFileQueue.Builder("childA2").build()) + .subQueue(new AllocationFileQueue.Builder("childA3").build()) + .subQueue(new AllocationFileQueue.Builder("childA4").build()) + .build()) + .subQueue(new AllocationFileQueue.Builder("parentB") + .weight(1) + .subQueue(new AllocationFileQueue.Builder("childB1").build()) + .subQueue(new AllocationFileQueue.Builder("childB2").build()) + .build()) + .build()) + .writeToFile(ALLOC_FILE); resourceManager = new MockRM(conf); resourceManager.start(); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFairSchedulerPreemption.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFairSchedulerPreemption.java index f86987f8b76..a5032cdc7bc 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFairSchedulerPreemption.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFairSchedulerPreemption.java @@ -26,6 +26,8 @@ import org.apache.hadoop.yarn.server.resourcemanager.scheduler.event.NodeUpdateSchedulerEvent; import org.apache.hadoop.yarn.server.resourcemanager.rmcontainer.RMContainer; import org.apache.hadoop.yarn.server.resourcemanager.rmcontainer.RMContainerImpl; +import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.allocationfile.AllocationFileQueue; +import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.allocationfile.AllocationFileWriter; import org.apache.hadoop.yarn.util.ControlledClock; import org.apache.hadoop.yarn.util.SystemClock; import org.junit.After; @@ -119,76 +121,69 @@ private void writeAllocFile() throws IOException { * |--- child-1 * |--- child-2 */ - PrintWriter out = new PrintWriter(new FileWriter(ALLOC_FILE)); - out.println(""); - out.println(""); - - out.println(""); - writePreemptionParams(out); - - // Child-1 - out.println(""); - writeResourceParams(out); - out.println(""); - - // Child-2 - out.println(""); - writeResourceParams(out); - out.println(""); - - out.println(""); // end of preemptable queue - - out.println(""); - writePreemptionParams(out); - out.println(""); - - // Queue with preemption disallowed - out.println(""); - out.println("false" + - ""); - writePreemptionParams(out); - - // Child-1 - out.println(""); - writeResourceParams(out); - out.println(""); - - // Child-2 - out.println(""); - writeResourceParams(out); - out.println(""); - - out.println(""); // end of nonpreemptable queue + AllocationFileWriter allocationFileWriter; + if (fairsharePreemption) { + allocationFileWriter = AllocationFileWriter.create() + .addQueue(new AllocationFileQueue.Builder("root") + .subQueue(new AllocationFileQueue.Builder("preemptable") + .fairSharePreemptionThreshold(1) + .fairSharePreemptionTimeout(0) + .subQueue(new AllocationFileQueue.Builder("child-1") + .build()) + .subQueue(new AllocationFileQueue.Builder("child-2") + .build()) + .build()) + .subQueue(new AllocationFileQueue.Builder("preemptable-sibling") + .fairSharePreemptionThreshold(1) + .fairSharePreemptionTimeout(0) + .build()) + .subQueue(new AllocationFileQueue.Builder("nonpreemptible") + .allowPreemptionFrom(false) + .fairSharePreemptionThreshold(1) + .fairSharePreemptionTimeout(0) + .subQueue(new AllocationFileQueue.Builder("child-1") + .build()) + .subQueue(new AllocationFileQueue.Builder("child-2") + .build()) + .build()) + .build()); + } else { + allocationFileWriter = AllocationFileWriter.create() + .addQueue(new AllocationFileQueue.Builder("root") + .subQueue(new AllocationFileQueue.Builder("preemptable") + .minSharePreemptionTimeout(0) + .subQueue(new AllocationFileQueue.Builder("child-1") + .minResources("4096mb,4vcores") + .build()) + .subQueue(new AllocationFileQueue.Builder("child-2") + .minResources("4096mb,4vcores") + .build()) + .build()) + .subQueue(new AllocationFileQueue.Builder("preemptable-sibling") + .minSharePreemptionTimeout(0) + .build()) + .subQueue(new AllocationFileQueue.Builder("nonpreemptible") + .allowPreemptionFrom(false) + .minSharePreemptionTimeout(0) + .subQueue(new AllocationFileQueue.Builder("child-1") + .minResources("4096mb,4vcores") + .build()) + .subQueue(new AllocationFileQueue.Builder("child-2") + .minResources("4096mb,4vcores") + .build()) + .build()) + .build()); + } if (drf) { - out.println("drf" + - ""); + allocationFileWriter.defaultQueueSchedulingPolicy("drf"); } - out.println(""); - out.close(); + allocationFileWriter.writeToFile(ALLOC_FILE.getAbsolutePath()); assertTrue("Allocation file does not exist, not running the test", ALLOC_FILE.exists()); } - private void writePreemptionParams(PrintWriter out) { - if (fairsharePreemption) { - out.println("1" + - ""); - out.println("0" + - ""); - } else { - out.println("0" + - ""); - } - } - - private void writeResourceParams(PrintWriter out) { - if (!fairsharePreemption) { - out.println("4096mb,4vcores"); - } - } - private void setupCluster() throws IOException { resourceManager = new MockRM(conf); scheduler = (FairScheduler) resourceManager.getResourceScheduler(); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFairSchedulerQueueACLs.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFairSchedulerQueueACLs.java index 32aab5531f8..ad56a209888 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFairSchedulerQueueACLs.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFairSchedulerQueueACLs.java @@ -18,41 +18,39 @@ package org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair; import java.io.File; -import java.io.FileWriter; -import java.io.IOException; -import java.io.PrintWriter; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.yarn.conf.YarnConfiguration; import org.apache.hadoop.yarn.server.resourcemanager.QueueACLsTestBase; +import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair + .allocationfile.AllocationFileQueue; +import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair + .allocationfile.AllocationFileWriter; public class TestFairSchedulerQueueACLs extends QueueACLsTestBase { @Override - protected Configuration createConfiguration() throws IOException { + protected Configuration createConfiguration() { FairSchedulerConfiguration fsConf = new FairSchedulerConfiguration(); - final String TEST_DIR = new File(System.getProperty("test.build.data", + final String testDir = new File(System.getProperty("test.build.data", "/tmp")).getAbsolutePath(); - final String ALLOC_FILE = new File(TEST_DIR, "test-queues.xml") + final String allocFile = new File(testDir, "test-queues.xml") .getAbsolutePath(); - PrintWriter out = new PrintWriter(new FileWriter(ALLOC_FILE)); - out.println(""); - out.println(""); - out.println(""); - out.println(" "); - out.println(" root_admin "); - out.println(" "); - out.println(" queueA_user,common_user "); - out.println(" queueA_admin "); - out.println(" "); - out.println(" "); - out.println(" queueB_user,common_user "); - out.println(" queueB_admin "); - out.println(" "); - out.println(""); - out.println(""); - out.close(); - fsConf.set(FairSchedulerConfiguration.ALLOCATION_FILE, ALLOC_FILE); + + AllocationFileWriter.create() + .addQueue(new AllocationFileQueue.Builder("root") + .aclSubmitApps(" ") + .aclAdministerApps("root_admin ") + .subQueue(new AllocationFileQueue.Builder("queueA") + .aclSubmitApps("queueA_user,common_user ") + .aclAdministerApps("queueA_admin ").build()) + .subQueue(new AllocationFileQueue.Builder("queueB") + .aclSubmitApps("queueB_user,common_user ") + .aclAdministerApps("queueB_admin ").build()) + .build()) + .writeToFile(allocFile); + + fsConf.set(FairSchedulerConfiguration.ALLOCATION_FILE, allocFile); fsConf.setBoolean(YarnConfiguration.YARN_ACL_ENABLE, true); fsConf.set(YarnConfiguration.RM_SCHEDULER, FairScheduler.class.getName()); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestQueueManagerRealScheduler.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestQueueManagerRealScheduler.java index 5736f75bbe5..e9a0b39dd82 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestQueueManagerRealScheduler.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestQueueManagerRealScheduler.java @@ -18,14 +18,16 @@ package org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair; import org.apache.hadoop.yarn.server.resourcemanager.MockRM; +import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair + .allocationfile.AllocationFileQueue; +import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair + .allocationfile.AllocationFileWriter; import org.junit.After; import org.junit.Before; import org.junit.Test; import java.io.File; -import java.io.FileWriter; import java.io.IOException; -import java.io.PrintWriter; import static org.junit.Assert.assertEquals; @@ -38,7 +40,7 @@ @Before public void setup() throws IOException { createConfiguration(); - writeAllocFile(30, 40); + writeAllocFile(30); conf.set(FairSchedulerConfiguration.ALLOCATION_FILE, ALLOC_FILE.getAbsolutePath()); @@ -56,32 +58,20 @@ public void teardown() { } } - private void writeAllocFile(int defaultFairShareTimeout, - int fairShareTimeout) throws IOException { - PrintWriter out = new PrintWriter(new FileWriter(ALLOC_FILE)); - out.println(""); - out.println(""); - out.println(""); - out.println(""); - out.println(""); - out.println(""); - out.println(""); - out.println(""); - out.println("5"); - out.println(""); - out.println(""); - out.println(""); - out.println(""); - out.println(""); - out.println(""); - out.println("15" - + ""); - out.println("" + - + defaultFairShareTimeout + ""); - out.println("" - + fairShareTimeout + ""); - out.println(""); - out.close(); + private void writeAllocFile(int defaultFairShareTimeout) { + AllocationFileWriter.create() + .addQueue(new AllocationFileQueue.Builder("default") + .build()) + .addQueue(new AllocationFileQueue.Builder("queueA").build()) + .addQueue(new AllocationFileQueue.Builder("queueB") + .subQueue(new AllocationFileQueue.Builder("queueB1") + .minSharePreemptionTimeout(5).build()) + .subQueue(new AllocationFileQueue.Builder("queueB2").build()) + .build()) + .addQueue(new AllocationFileQueue.Builder("queueC").build()) + .defaultMinSharePreemptionTimeout(15) + .defaultFairSharePreemptionTimeout(defaultFairShareTimeout) + .writeToFile(ALLOC_FILE.getAbsolutePath()); } @Test @@ -120,7 +110,7 @@ public void testBackwardsCompatiblePreemptionConfiguration() // Lower the fairshare preemption timeouts and verify it is picked // correctly. - writeAllocFile(25, 30); + writeAllocFile(25); scheduler.reinitialize(conf, resourceManager.getRMContext()); assertEquals(25000, queueMgr.getQueue("root") .getFairSharePreemptionTimeout()); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestSchedulingPolicy.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestSchedulingPolicy.java index e2be0d95adc..4546355a994 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestSchedulingPolicy.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestSchedulingPolicy.java @@ -19,9 +19,7 @@ package org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair; import java.io.File; -import java.io.FileWriter; import java.io.IOException; -import java.io.PrintWriter; import java.util.Collection; import java.util.Comparator; import java.util.Stack; @@ -32,6 +30,10 @@ import org.apache.hadoop.yarn.api.records.Resource; import org.apache.hadoop.yarn.server.resourcemanager.RMContext; import org.apache.hadoop.yarn.server.resourcemanager.placement.PlacementManager; +import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair + .allocationfile.AllocationFileQueue; +import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair + .allocationfile.AllocationFileWriter; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.policies.DominantResourceFairnessPolicy; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.policies.FairSharePolicy; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.policies.FifoPolicy; @@ -319,22 +321,16 @@ public boolean isPreemptable() { @Test public void testSchedulingPolicyViolation() throws IOException { conf.set(FairSchedulerConfiguration.ALLOCATION_FILE, ALLOC_FILE); - PrintWriter out = new PrintWriter(new FileWriter(ALLOC_FILE)); - out.println(""); - out.println(""); - out.println(""); - out.println("fair"); - out.println(" "); - out.println(" drf"); - out.println(" "); - out.println(" "); - out.println(" fair"); - out.println(" "); - out.println(""); - out.println("drf" + - ""); - out.println(""); - out.close(); + AllocationFileWriter.create() + .defaultQueueSchedulingPolicy("drf") + .addQueue(new AllocationFileQueue.Builder("root") + .schedulingPolicy("fair") + .subQueue(new AllocationFileQueue.Builder("child1") + .schedulingPolicy("drf").build()) + .subQueue(new AllocationFileQueue.Builder("child2") + .schedulingPolicy("fair").build()) + .build()) + .writeToFile(ALLOC_FILE); scheduler.init(conf); @@ -349,22 +345,16 @@ public void testSchedulingPolicyViolation() throws IOException { + " policy if its parent policy is 'fair'.", dynamicQueue); // Set child1 to 'fair' and child2 to 'drf', the reload the allocation file. - out = new PrintWriter(new FileWriter(ALLOC_FILE)); - out.println(""); - out.println(""); - out.println(""); - out.println("fair"); - out.println(" "); - out.println(" fair"); - out.println(" "); - out.println(" "); - out.println(" drf"); - out.println(" "); - out.println(""); - out.println("drf" + - ""); - out.println(""); - out.close(); + AllocationFileWriter.create() + .defaultQueueSchedulingPolicy("drf") + .addQueue(new AllocationFileQueue.Builder("root") + .schedulingPolicy("fair") + .subQueue(new AllocationFileQueue.Builder("child1") + .schedulingPolicy("fair").build()) + .subQueue(new AllocationFileQueue.Builder("child2") + .schedulingPolicy("drf").build()) + .build()) + .writeToFile(ALLOC_FILE); scheduler.reinitialize(conf, null); child1 = scheduler.getQueueManager().getQueue("child1"); @@ -379,26 +369,21 @@ public void testSchedulingPolicyViolation() throws IOException { } @Test - public void testSchedulingPolicyViolationInTheMiddleLevel() - throws IOException { + public void testSchedulingPolicyViolationInTheMiddleLevel() { conf.set(FairSchedulerConfiguration.ALLOCATION_FILE, ALLOC_FILE); - PrintWriter out = new PrintWriter(new FileWriter(ALLOC_FILE)); - out.println(""); - out.println(""); - out.println(""); - out.println("fair"); - out.println(" "); - out.println(" fair"); - out.println(" "); - out.println(" drf"); - out.println(" "); - out.println(" fair"); - out.println(" "); - out.println(" "); - out.println(" "); - out.println(""); - out.println(""); - out.close(); + AllocationFileWriter.create() + .addQueue(new AllocationFileQueue.Builder("root") + .schedulingPolicy("fair") + .subQueue(new AllocationFileQueue.Builder("level2") + .schedulingPolicy("fair") + .subQueue(new AllocationFileQueue.Builder("level3") + .schedulingPolicy("drf") + .subQueue(new AllocationFileQueue.Builder("leaf") + .schedulingPolicy("fair").build()) + .build()) + .build()) + .build()) + .writeToFile(ALLOC_FILE); scheduler.init(conf); @@ -417,19 +402,16 @@ public void testSchedulingPolicyViolationInTheMiddleLevel() public void testFIFOPolicyOnlyForLeafQueues() throws IOException { conf.set(FairSchedulerConfiguration.ALLOCATION_FILE, ALLOC_FILE); - PrintWriter out = new PrintWriter(new FileWriter(ALLOC_FILE)); - out.println(""); - out.println(""); - out.println(""); - out.println(" "); - out.println(" fifo"); - out.println(" "); - out.println(" fair"); - out.println(" "); - out.println(" "); - out.println(""); - out.println(""); - out.close(); + + AllocationFileWriter.create() + .addQueue(new AllocationFileQueue.Builder("root") + .subQueue(new AllocationFileQueue.Builder("intermediate") + .schedulingPolicy("fifo") + .subQueue(new AllocationFileQueue.Builder("leaf") + .schedulingPolicy("fair").build()) + .build()) + .build()) + .writeToFile(ALLOC_FILE); scheduler.init(conf); @@ -437,19 +419,15 @@ public void testFIFOPolicyOnlyForLeafQueues() assertNull("Queue 'intermediate' should be null since 'fifo' is only for " + "leaf queue.", intermediate); - out = new PrintWriter(new FileWriter(ALLOC_FILE)); - out.println(""); - out.println(""); - out.println(""); - out.println(" "); - out.println(" fair"); - out.println(" "); - out.println(" fifo"); - out.println(" "); - out.println(" "); - out.println(""); - out.println(""); - out.close(); + AllocationFileWriter.create() + .addQueue(new AllocationFileQueue.Builder("root") + .subQueue(new AllocationFileQueue.Builder("intermediate") + .schedulingPolicy("fair") + .subQueue(new AllocationFileQueue.Builder("leaf") + .schedulingPolicy("fifo").build()) + .build()) + .build()) + .writeToFile(ALLOC_FILE); scheduler.reinitialize(conf, null); @@ -463,39 +441,28 @@ public void testFIFOPolicyOnlyForLeafQueues() @Test public void testPolicyReinitilization() throws IOException { conf.set(FairSchedulerConfiguration.ALLOCATION_FILE, ALLOC_FILE); - PrintWriter out = new PrintWriter(new FileWriter(ALLOC_FILE)); - out.println(""); - out.println(""); - out.println(""); - out.println("fair"); - out.println(" "); - out.println(" fair"); - out.println(" "); - out.println(" "); - out.println(" fair"); - out.println(" "); - out.println(""); - out.println(""); - out.close(); + AllocationFileWriter.create() + .addQueue(new AllocationFileQueue.Builder("root") + .schedulingPolicy("fair") + .subQueue(new AllocationFileQueue.Builder("child1") + .schedulingPolicy("fair").build()) + .subQueue(new AllocationFileQueue.Builder("child2") + .schedulingPolicy("fair").build()) + .build()) + .writeToFile(ALLOC_FILE); scheduler.init(conf); // Set child1 to 'drf' which is not allowed, then reload the allocation file - out = new PrintWriter(new FileWriter(ALLOC_FILE)); - out.println(""); - out.println(""); - out.println(""); - out.println("fair"); - out.println(" "); - out.println(" drf"); - out.println(" "); - out.println(" "); - out.println(" fifo"); - out.println(" "); - out.println(""); - out.println(""); - out.close(); - + AllocationFileWriter.create() + .addQueue(new AllocationFileQueue.Builder("root") + .schedulingPolicy("fair") + .subQueue(new AllocationFileQueue.Builder("child1") + .schedulingPolicy("drf").build()) + .subQueue(new AllocationFileQueue.Builder("child2") + .schedulingPolicy("fifo").build()) + .build()) + .writeToFile(ALLOC_FILE); scheduler.reinitialize(conf, null); FSQueue child1 = scheduler.getQueueManager().getQueue("child1"); @@ -508,20 +475,15 @@ public void testPolicyReinitilization() throws IOException { child2.getPolicy() instanceof FairSharePolicy); // Set both child1 and root to 'drf', then reload the allocation file - out = new PrintWriter(new FileWriter(ALLOC_FILE)); - out.println(""); - out.println(""); - out.println(""); - out.println("drf"); - out.println(" "); - out.println(" drf"); - out.println(" "); - out.println(" "); - out.println(" fifo"); - out.println(" "); - out.println(""); - out.println(""); - out.close(); + AllocationFileWriter.create() + .addQueue(new AllocationFileQueue.Builder("root") + .schedulingPolicy("drf") + .subQueue(new AllocationFileQueue.Builder("child1") + .schedulingPolicy("drf").build()) + .subQueue(new AllocationFileQueue.Builder("child2") + .schedulingPolicy("fifo").build()) + .build()) + .writeToFile(ALLOC_FILE); scheduler.reinitialize(conf, null); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/allocationfile/AllocationFileQueue.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/allocationfile/AllocationFileQueue.java index db816130e0e..ebf0616b2e6 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/allocationfile/AllocationFileQueue.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/allocationfile/AllocationFileQueue.java @@ -16,69 +16,261 @@ package org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.allocationfile; +import com.google.common.collect.Lists; + import java.io.PrintWriter; import java.io.StringWriter; import java.util.List; -class AllocationFileQueue { - private final AllocationFileQueueProperties properties; +import static org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair + .allocationfile.AllocationFileWriter.addIfPresent; + +/** + * DAO for Allocation File Queue. + */ +final public class AllocationFileQueue { + private static final String DEFAULT_TAG_NAME = "queue"; + private static final String LEGACY_TAG_NAME = "pool"; + + private final String queueName; + private final String minResources; + private final String maxResources; + private final String aclAdministerApps; + private final String aclSubmitApps; + private final String aclSubmitReservations; + private final String aclAdministerReservations; + private final String aclListReservations; + private final String schedulingPolicy; + private final Integer maxRunningApps; + private final Double maxAMShare; + private final Boolean allowPreemptionFrom; + private final Integer minSharePreemptionTimeout; + private final String maxChildResources; + private final Integer fairSharePreemptionTimeout; + private final Double fairSharePreemptionThreshold; + private final String maxContainerAllocation; private final List subQueues; + private final Float weight; + private String tagName; - AllocationFileQueue(AllocationFileQueueProperties properties, - List subQueues) { - this.properties = properties; - this.subQueues = subQueues; + private final boolean parent; + private final boolean reservation; + + private AllocationFileQueue(Builder builder) { + this.queueName = builder.name; + this.parent = builder.parent; + this.minResources = builder.minResources; + this.maxResources = builder.maxResources; + this.aclAdministerApps = builder.aclAdministerApps; + this.aclSubmitApps = builder.aclSubmitApps; + this.aclSubmitReservations = builder.aclSubmitReservations; + this.aclAdministerReservations = builder.aclAdministerReservations; + this.aclListReservations = builder.aclListReservations; + this.schedulingPolicy = builder.schedulingPolicy; + this.maxRunningApps = builder.maxRunningApps; + this.maxAMShare = builder.maxAMShare; + this.allowPreemptionFrom = builder.allowPreemptionFrom; + this.minSharePreemptionTimeout = builder.minSharePreemptionTimeout; + this.maxChildResources = builder.maxChildResources; + this.fairSharePreemptionTimeout = builder.fairSharePreemptionTimeout; + this.fairSharePreemptionThreshold = builder.fairSharePreemptionThreshold; + this.maxContainerAllocation = builder.maxContainerAllocation; + this.weight = builder.weight; + this.reservation = builder.reservation; + this.subQueues = builder.subQueues; + this.tagName = DEFAULT_TAG_NAME; } String render() { StringWriter sw = new StringWriter(); PrintWriter pw = new PrintWriter(sw); printStartTag(pw); - AllocationFileWriter.printQueues(pw, subQueues); - AllocationFileWriter.addIfPresent(pw, "minResources", - properties::getMinResources); - AllocationFileWriter.addIfPresent(pw, "maxResources", - properties::getMaxResources); - AllocationFileWriter.addIfPresent(pw, "aclAdministerApps", - properties::getAclAdministerApps); - AllocationFileWriter.addIfPresent(pw, "aclSubmitApps", - properties::getAclSubmitApps); - AllocationFileWriter.addIfPresent(pw, "schedulingPolicy", - properties::getSchedulingPolicy); - AllocationFileWriter.addIfPresent(pw, "maxRunningApps", - () -> AllocationFileWriter - .createNumberSupplier(properties.getMaxRunningApps())); - AllocationFileWriter.addIfPresent(pw, "maxAMShare", - () -> AllocationFileWriter.createNumberSupplier(properties - .getMaxAMShare())); - AllocationFileWriter.addIfPresent(pw, "minSharePreemptionTimeout", - () -> AllocationFileWriter - .createNumberSupplier(properties.getMinSharePreemptionTimeout())); - AllocationFileWriter.addIfPresent(pw, "maxChildResources", - properties::getMaxChildResources); - AllocationFileWriter.addIfPresent(pw, "fairSharePreemptionTimeout", - () -> AllocationFileWriter - .createNumberSupplier(properties.getFairSharePreemptionTimeout())); - AllocationFileWriter.addIfPresent(pw, "fairSharePreemptionThreshold", - () -> AllocationFileWriter.createNumberSupplier( - properties.getFairSharePreemptionThreshold())); - AllocationFileWriter.addIfPresent(pw, "maxContainerAllocation", - () -> AllocationFileWriter - .createNumberSupplier(properties.getMaxContainerAllocation())); + AllocationFileWriter.printQueues(pw, subQueues, + tagName.equals(LEGACY_TAG_NAME)); + addIfPresent(pw, "minResources", minResources); + addIfPresent(pw, "maxResources", maxResources); + addIfPresent(pw, "aclAdministerApps", aclAdministerApps); + addIfPresent(pw, "aclSubmitApps", aclSubmitApps); + addIfPresent(pw, "aclSubmitReservations", aclSubmitReservations); + addIfPresent(pw, "aclAdministerReservations", aclAdministerReservations); + addIfPresent(pw, "aclListReservations", aclListReservations); + addIfPresent(pw, "schedulingPolicy", schedulingPolicy); + addIfPresent(pw, "maxRunningApps", maxRunningApps); + addIfPresent(pw, "maxAMShare", maxAMShare); + addIfPresent(pw, "allowPreemptionFrom", allowPreemptionFrom); + addIfPresent(pw, "minSharePreemptionTimeout", minSharePreemptionTimeout); + addIfPresent(pw, "maxChildResources", maxChildResources); + addIfPresent(pw, "fairSharePreemptionTimeout", fairSharePreemptionTimeout); + addIfPresent(pw, "fairSharePreemptionThreshold", + fairSharePreemptionThreshold); + addIfPresent(pw, "maxContainerAllocation", maxContainerAllocation); + addIfPresent(pw, "weight", weight); + if (reservation) { + pw.println(""); + } printEndTag(pw); pw.close(); + return sw.toString(); } + String renderWithLegacyTag() { + this.tagName = LEGACY_TAG_NAME; + return render(); + } + private void printStartTag(PrintWriter pw) { - pw.print(""); } private void printEndTag(PrintWriter pw) { - pw.println(""); + pw.println(""); + } + + /** + * Class that can build queues (with subqueues) for testcases. + * The intention of having this class to group the common properties of + * simple queues and subqueues by methods delegating calls to a + * queuePropertiesBuilder instance. + */ + public static class Builder { + private String name; + private String minResources; + private String maxResources; + private String aclAdministerApps; + private String aclSubmitApps; + private String aclSubmitReservations; + private String aclAdministerReservations; + private String aclListReservations; + private String schedulingPolicy; + private Integer maxRunningApps; + private Double maxAMShare; + private Boolean allowPreemptionFrom; + private Integer minSharePreemptionTimeout; + private boolean parent; + private String maxChildResources; + private Integer fairSharePreemptionTimeout; + private Double fairSharePreemptionThreshold; + private String maxContainerAllocation; + private boolean reservation; + private final List subQueues = Lists.newArrayList(); + private Float weight; + + public Builder(String name) { + this.name = name; + } + + public Builder parent(boolean value) { + this.parent = value; + return this; + } + + public Builder minResources(String value) { + this.minResources = value; + return this; + } + + public Builder maxResources(String value) { + this.maxResources = value; + return this; + } + + public Builder aclAdministerApps(String value) { + this.aclAdministerApps = value; + return this; + } + + public Builder aclSubmitApps(String value) { + this.aclSubmitApps = value; + return this; + } + + public Builder aclSubmitReservations(String value) { + this.aclSubmitReservations = value; + return this; + } + + public Builder aclAdministerReservations(String value) { + this.aclAdministerReservations = value; + return this; + } + + public Builder aclListReservations(String value) { + this.aclListReservations = value; + return this; + } + + public Builder schedulingPolicy(String value) { + this.schedulingPolicy = value; + return this; + } + + public Builder maxRunningApps(int value) { + this.maxRunningApps = value; + return this; + } + + public Builder maxAMShare(double value) { + this.maxAMShare = value; + return this; + } + + public Builder allowPreemptionFrom(boolean value) { + this.allowPreemptionFrom = value; + return this; + } + + public Builder minSharePreemptionTimeout(int value) { + this.minSharePreemptionTimeout = value; + return this; + } + + public Builder maxChildResources(String value) { + this.maxChildResources = value; + return this; + } + + public Builder fairSharePreemptionTimeout(Integer value) { + this.fairSharePreemptionTimeout = value; + return this; + } + + public Builder fairSharePreemptionThreshold( + double value) { + this.fairSharePreemptionThreshold = value; + return this; + } + + public Builder maxContainerAllocation(String value) { + this.maxContainerAllocation = value; + return this; + } + + public Builder weight(float value) { + this.weight = value; + return this; + } + + public Builder reservation() { + this.reservation = true; + return this; + } + + public Builder subQueue(AllocationFileQueue queue) { + if (queue == null) { + throw new IllegalArgumentException("Subqueue cannot be null!"); + } + subQueues.add(queue); + return this; + } + + public AllocationFileQueue build() { + return new AllocationFileQueue(this); + } } } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/allocationfile/AllocationFileQueueBuilder.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/allocationfile/AllocationFileQueueBuilder.java deleted file mode 100644 index 176024e9b46..00000000000 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/allocationfile/AllocationFileQueueBuilder.java +++ /dev/null @@ -1,121 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * http://www.apache.org/licenses/LICENSE-2.0 - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.allocationfile; - -/** - * Abstract base class for building simple queues and subqueues for testcases. - * Currently there are two concrete types subclassed from this class: - * {@link AllocationFileSimpleQueueBuilder} and - * {@link AllocationFileSubQueueBuilder}. - * The intention of having this class to group the common properties of - * simple queues and subqueues by methods delegating calls to a - * queuePropertiesBuilder instance. - */ -public abstract class AllocationFileQueueBuilder { - final AllocationFileQueueProperties.Builder queuePropertiesBuilder; - - AllocationFileQueueBuilder() { - this.queuePropertiesBuilder = - AllocationFileQueueProperties.Builder.create(); - } - - public AllocationFileQueueBuilder parent(boolean parent) { - this.queuePropertiesBuilder.parent(parent); - return this; - } - - public AllocationFileQueueBuilder minResources(String value) { - this.queuePropertiesBuilder.minResources(value); - return this; - } - - public AllocationFileQueueBuilder maxResources(String value) { - this.queuePropertiesBuilder.maxResources(value); - return this; - } - - public AllocationFileQueueBuilder aclAdministerApps(String value) { - this.queuePropertiesBuilder.aclAdministerApps(value); - return this; - } - - public AllocationFileQueueBuilder aclSubmitApps(String value) { - this.queuePropertiesBuilder.aclSubmitApps(value); - return this; - } - - public AllocationFileQueueBuilder schedulingPolicy(String value) { - this.queuePropertiesBuilder.schedulingPolicy(value); - return this; - } - - public AllocationFileQueueBuilder maxRunningApps(int value) { - this.queuePropertiesBuilder.maxRunningApps(value); - return this; - } - - public AllocationFileQueueBuilder maxAMShare(double value) { - this.queuePropertiesBuilder.maxAMShare(value); - return this; - } - - public AllocationFileQueueBuilder minSharePreemptionTimeout(int value) { - this.queuePropertiesBuilder.minSharePreemptionTimeout(value); - return this; - } - - public AllocationFileQueueBuilder maxChildResources(String value) { - this.queuePropertiesBuilder.maxChildResources(value); - return this; - } - - public AllocationFileQueueBuilder fairSharePreemptionTimeout(Integer value) { - this.queuePropertiesBuilder.fairSharePreemptionTimeout(value); - return this; - } - - public AllocationFileQueueBuilder fairSharePreemptionThreshold( - double value) { - this.queuePropertiesBuilder.fairSharePreemptionThreshold(value); - return this; - } - - public AllocationFileQueueBuilder maxContainerAllocation( - String maxContainerAllocation) { - this.queuePropertiesBuilder.maxContainerAllocation(maxContainerAllocation); - return this; - } - - public AllocationFileQueueBuilder subQueue(String queueName) { - if (this instanceof AllocationFileSimpleQueueBuilder) { - return new AllocationFileSubQueueBuilder( - (AllocationFileSimpleQueueBuilder) this, queueName); - } else { - throw new IllegalStateException( - "subQueue can only be invoked on instances of " - + AllocationFileSimpleQueueBuilder.class); - } - } - - public abstract AllocationFileWriter buildQueue(); - - public abstract AllocationFileSimpleQueueBuilder buildSubQueue(); - - AllocationFileQueueProperties.Builder getqueuePropertiesBuilder() { - return queuePropertiesBuilder; - } -} diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/allocationfile/AllocationFileQueuePlacementPolicy.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/allocationfile/AllocationFileQueuePlacementPolicy.java new file mode 100644 index 00000000000..48aab06ef34 --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/allocationfile/AllocationFileQueuePlacementPolicy.java @@ -0,0 +1,63 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * http://www.apache.org/licenses/LICENSE-2.0 + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.allocationfile; + + +import com.google.common.collect.Lists; + +import java.io.PrintWriter; +import java.io.StringWriter; +import java.util.List; + +/** + * Helper class to manage {@link AllocationFileQueuePlacementRule} + * instances for {@link AllocationFileWriter}. + */ +public class AllocationFileQueuePlacementPolicy { + private List rules = Lists.newArrayList(); + + public AllocationFileQueuePlacementPolicy addRule( + AllocationFileQueuePlacementRule rule) { + this.rules.add(rule); + return this; + } + + public String render() { + StringWriter sw = new StringWriter(); + PrintWriter pw = new PrintWriter(sw); + addStartTag(pw); + addRules(pw); + addEndTag(pw); + pw.close(); + + return sw.toString(); + } + + private void addStartTag(PrintWriter pw) { + pw.println(""); + } + + private void addRules(PrintWriter pw) { + for (AllocationFileQueuePlacementRule rule : rules) { + pw.println(rule.render()); + } + } + + private void addEndTag(PrintWriter pw) { + pw.println(""); + } +} diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/allocationfile/AllocationFileQueuePlacementRule.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/allocationfile/AllocationFileQueuePlacementRule.java new file mode 100644 index 00000000000..cfc382fbf35 --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/allocationfile/AllocationFileQueuePlacementRule.java @@ -0,0 +1,88 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * http://www.apache.org/licenses/LICENSE-2.0 + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.allocationfile; + + +import com.google.common.collect.Lists; + +import java.io.PrintWriter; +import java.io.StringWriter; +import java.util.List; + +/** + * Helper class for {@link AllocationFileWriter} to manage + * queue placement rules. + */ +public class AllocationFileQueuePlacementRule { + private String name; + private boolean create = true; + private String queue; + private List nestedRules = Lists + .newArrayList(); + + public AllocationFileQueuePlacementRule(String name) { + this.name = name; + } + + public AllocationFileQueuePlacementRule create(boolean shouldCreate) { + this.create = shouldCreate; + return this; + } + + public AllocationFileQueuePlacementRule queue(String selectedQueue) { + this.queue = selectedQueue; + return this; + } + + public AllocationFileQueuePlacementRule addNestedRule( + AllocationFileQueuePlacementRule rule) { + this.nestedRules.add(rule); + return this; + } + + public String render() { + StringWriter sw = new StringWriter(); + PrintWriter pw = new PrintWriter(sw); + addStartTag(pw); + addNestedRules(pw); + addEndTag(pw); + pw.close(); + + return sw.toString(); + } + + private void addStartTag(PrintWriter pw) { + pw.print(""); + } + + private void addNestedRules(PrintWriter pw) { + if (nestedRules != null && !nestedRules.isEmpty()) { + for (AllocationFileQueuePlacementRule rule : nestedRules) { + pw.println(rule.render()); + } + } + } + + private void addEndTag(PrintWriter pw) { + pw.println(""); + } +} diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/allocationfile/AllocationFileQueueProperties.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/allocationfile/AllocationFileQueueProperties.java deleted file mode 100644 index 0a0f330b88c..00000000000 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/allocationfile/AllocationFileQueueProperties.java +++ /dev/null @@ -1,214 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * http://www.apache.org/licenses/LICENSE-2.0 - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.allocationfile; - -/** - * The purpose of this class is to store all properties of a queue. - */ -public class AllocationFileQueueProperties { - private final String queueName; - private final String minResources; - private final String maxResources; - private final String aclAdministerApps; - private final String aclSubmitApps; - private final String schedulingPolicy; - private final Integer maxRunningApps; - private final Double maxAMShare; - private final Integer minSharePreemptionTimeout; - private final Boolean parent; - private final String maxChildResources; - private final Integer fairSharePreemptionTimeout; - private final Double fairSharePreemptionThreshold; - private final String maxContainerAllocation; - - AllocationFileQueueProperties(Builder builder) { - this.queueName = builder.queueName; - this.parent = builder.parent; - this.minResources = builder.minResources; - this.maxResources = builder.maxResources; - this.aclAdministerApps = builder.aclAdministerApps; - this.aclSubmitApps = builder.aclSubmitApps; - this.schedulingPolicy = builder.schedulingPolicy; - this.maxRunningApps = builder.maxRunningApps; - this.maxAMShare = builder.maxAMShare; - this.minSharePreemptionTimeout = builder.minSharePreemptionTimeout; - this.maxChildResources = builder.maxChildResources; - this.fairSharePreemptionTimeout = builder.fairSharePreemptionTimeout; - this.fairSharePreemptionThreshold = builder.fairSharePreemptionThreshold; - this.maxContainerAllocation = builder.maxContainerAllocation; - } - - public String getQueueName() { - return queueName; - } - - public String getMinResources() { - return minResources; - } - - public String getMaxResources() { - return maxResources; - } - - public String getAclAdministerApps() { - return aclAdministerApps; - } - - public String getAclSubmitApps() { - return aclSubmitApps; - } - - public String getSchedulingPolicy() { - return schedulingPolicy; - } - - public Integer getMaxRunningApps() { - return maxRunningApps; - } - - public Double getMaxAMShare() { - return maxAMShare; - } - - public Integer getMinSharePreemptionTimeout() { - return minSharePreemptionTimeout; - } - - public Boolean getParent() { - return parent; - } - - public String getMaxChildResources() { - return maxChildResources; - } - - public Integer getFairSharePreemptionTimeout() { - return fairSharePreemptionTimeout; - } - - public Double getFairSharePreemptionThreshold() { - return fairSharePreemptionThreshold; - } - - public String getMaxContainerAllocation() { - return maxContainerAllocation; - } - - /** - * Builder class for {@link AllocationFileQueueProperties}. - */ - public static final class Builder { - private String queueName; - private Boolean parent = false; - private String minResources; - private String maxResources; - private String aclAdministerApps; - private String aclSubmitApps; - private String schedulingPolicy; - private Integer maxRunningApps; - private Double maxAMShare; - private Integer minSharePreemptionTimeout; - private String maxChildResources; - private Integer fairSharePreemptionTimeout; - private Double fairSharePreemptionThreshold; - private String maxContainerAllocation; - - Builder() { - } - - public static Builder create() { - return new Builder(); - } - - public Builder queueName(String queueName) { - this.queueName = queueName; - return this; - } - - public Builder minResources(String minResources) { - this.minResources = minResources; - return this; - } - - public Builder maxResources(String maxResources) { - this.maxResources = maxResources; - return this; - } - - public Builder aclAdministerApps(String aclAdministerApps) { - this.aclAdministerApps = aclAdministerApps; - return this; - } - - public Builder aclSubmitApps(String aclSubmitApps) { - this.aclSubmitApps = aclSubmitApps; - return this; - } - - public Builder schedulingPolicy(String schedulingPolicy) { - this.schedulingPolicy = schedulingPolicy; - return this; - } - - public Builder maxRunningApps(Integer maxRunningApps) { - this.maxRunningApps = maxRunningApps; - return this; - } - - public Builder maxAMShare(Double maxAMShare) { - this.maxAMShare = maxAMShare; - return this; - } - - public Builder maxContainerAllocation(String maxContainerAllocation) { - this.maxContainerAllocation = maxContainerAllocation; - return this; - } - - public Builder minSharePreemptionTimeout( - Integer minSharePreemptionTimeout) { - this.minSharePreemptionTimeout = minSharePreemptionTimeout; - return this; - } - - public Builder parent(Boolean parent) { - this.parent = parent; - return this; - } - - public Builder maxChildResources(String maxChildResources) { - this.maxChildResources = maxChildResources; - return this; - } - - public Builder fairSharePreemptionTimeout( - Integer fairSharePreemptionTimeout) { - this.fairSharePreemptionTimeout = fairSharePreemptionTimeout; - return this; - } - - public Builder fairSharePreemptionThreshold( - Double fairSharePreemptionThreshold) { - this.fairSharePreemptionThreshold = fairSharePreemptionThreshold; - return this; - } - - public AllocationFileQueueProperties build() { - return new AllocationFileQueueProperties(this); - } - } -} diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/allocationfile/AllocationFileSimpleQueueBuilder.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/allocationfile/AllocationFileSimpleQueueBuilder.java deleted file mode 100644 index 93d100ed35c..00000000000 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/allocationfile/AllocationFileSimpleQueueBuilder.java +++ /dev/null @@ -1,64 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * http://www.apache.org/licenses/LICENSE-2.0 - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.allocationfile; - -import java.util.ArrayList; -import java.util.List; - -/** - * Queue builder that can build a simple queue with its properties. - * Subqueues can be added with {@link #addSubQueue(AllocationFileQueue)}. - */ -public class AllocationFileSimpleQueueBuilder - extends AllocationFileQueueBuilder { - private final AllocationFileWriter allocationFileWriter; - private final List subQueues = new ArrayList<>(); - - AllocationFileSimpleQueueBuilder(AllocationFileWriter allocationFileWriter, - String queueName) { - this.allocationFileWriter = allocationFileWriter; - getqueuePropertiesBuilder().queueName(queueName); - } - - void addSubQueue(AllocationFileQueue queue) { - subQueues.add(queue); - } - - @Override - public AllocationFileWriter buildQueue() { - AllocationFileQueueProperties queueProperties = - getqueuePropertiesBuilder().build(); - AllocationFileQueue queue = - new AllocationFileQueue(queueProperties, subQueues); - - if (allocationFileWriter != null) { - allocationFileWriter.addQueue(queue); - } else { - throw new IllegalStateException( - "allocationFileWriter field has to be set on a " + getClass()); - } - - return allocationFileWriter; - } - - @Override - public AllocationFileSimpleQueueBuilder buildSubQueue() { - throw new IllegalStateException( - "buildSubQueue is not supported in " + getClass()); - } - -} diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/allocationfile/AllocationFileSubQueueBuilder.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/allocationfile/AllocationFileSubQueueBuilder.java deleted file mode 100644 index 728aedcbc82..00000000000 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/allocationfile/AllocationFileSubQueueBuilder.java +++ /dev/null @@ -1,54 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * http://www.apache.org/licenses/LICENSE-2.0 - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.allocationfile; - -import com.google.common.collect.Lists; - - -/** - * Queue builder that can build a subqueue with its properties. - */ -public class AllocationFileSubQueueBuilder extends AllocationFileQueueBuilder { - private AllocationFileSimpleQueueBuilder parentQueueBuilder; - - AllocationFileSubQueueBuilder( - AllocationFileSimpleQueueBuilder parentQueueBuilder, String queueName) { - getqueuePropertiesBuilder().queueName(queueName); - this.parentQueueBuilder = parentQueueBuilder; - } - - @Override - public AllocationFileWriter buildQueue() { - throw new IllegalStateException( - "BuildQueue is not supported in " + getClass()); - } - - public AllocationFileSimpleQueueBuilder buildSubQueue() { - AllocationFileQueueProperties queueProperties = - getqueuePropertiesBuilder().build(); - AllocationFileQueue queue = - new AllocationFileQueue(queueProperties, Lists.newArrayList()); - - if (parentQueueBuilder != null) { - parentQueueBuilder.addSubQueue(queue); - return parentQueueBuilder; - } else { - throw new IllegalStateException( - "parentQueueBuilder field has to be set on a " + getClass()); - } - } -} diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/allocationfile/AllocationFileWriter.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/allocationfile/AllocationFileWriter.java index df1cc53d840..b78bfaeb36a 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/allocationfile/AllocationFileWriter.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/allocationfile/AllocationFileWriter.java @@ -16,12 +16,10 @@ package org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.allocationfile; -import java.io.FileWriter; import java.io.IOException; import java.io.PrintWriter; import java.util.ArrayList; import java.util.List; -import java.util.function.Supplier; /** * This class is capable of serializing allocation file data to a file @@ -39,6 +37,10 @@ private String defaultQueueSchedulingPolicy; private List queues = new ArrayList<>(); private UserSettings userSettings; + private boolean useLegacyTagNameForQueues = false; + private String reservationAgent; + private String reservationPolicy; + private AllocationFileQueuePlacementPolicy queuePlacementPolicy; private AllocationFileWriter() { } @@ -47,8 +49,9 @@ public static AllocationFileWriter create() { return new AllocationFileWriter(); } - public AllocationFileSimpleQueueBuilder queue(String queueName) { - return new AllocationFileSimpleQueueBuilder(this, queueName); + public AllocationFileWriter addQueue(AllocationFileQueue queue) { + queues.add(queue); + return this; } public AllocationFileWriter queueMaxAppsDefault(int value) { @@ -92,21 +95,42 @@ public AllocationFileWriter defaultQueueSchedulingPolicy(String value) { return this; } - public UserSettings.Builder userSettings(String username) { - return new UserSettings.Builder(this, username); + public AllocationFileWriter useLegacyTagNameForQueues() { + this.useLegacyTagNameForQueues = true; + return this; + } + + public AllocationFileWriter reservationAgent(String value) { + this.reservationAgent = value; + return this; + } + + public AllocationFileWriter reservationPolicy(String value) { + this.reservationPolicy = value; + return this; } - void addQueue(AllocationFileQueue queue) { - this.queues.add(queue); + public AllocationFileWriter userSettings(UserSettings settings) { + this.userSettings = settings; + return this; } - void setUserSettings(UserSettings userSettings) { - this.userSettings = userSettings; + public AllocationFileWriter queuePlacementPolicy( + AllocationFileQueuePlacementPolicy policy) { + this.queuePlacementPolicy = policy; + return this; } - static void printQueues(PrintWriter pw, List queues) { + static void printQueues(PrintWriter pw, List queues, + boolean useLegacyTagName) { for (AllocationFileQueue queue : queues) { - pw.println(queue.render()); + final String queueStr; + if (useLegacyTagName) { + queueStr = queue.renderWithLegacyTag(); + } else { + queueStr = queue.render(); + } + pw.println(queueStr); } } @@ -114,22 +138,18 @@ private void printUserSettings(PrintWriter pw) { pw.println(userSettings.render()); } - static void addIfPresent(PrintWriter pw, String tag, - Supplier supplier) { - if (supplier.get() != null) { - pw.println("<" + tag + ">" + supplier.get() + ""); - } + private void printQueuePlacementPolicy(PrintWriter pw) { + pw.println(queuePlacementPolicy.render()); } - static String createNumberSupplier(Object number) { - if (number != null) { - return number.toString(); + static void addIfPresent(PrintWriter pw, String tag, Object obj) { + if (obj != null) { + pw.println("<" + tag + ">" + obj.toString() + ""); } - return null; } private void writeHeader(PrintWriter pw) { - pw.println(""); + pw.println(""); pw.println(""); } @@ -140,34 +160,37 @@ private void writeFooter(PrintWriter pw) { public void writeToFile(String filename) { PrintWriter pw; try { - pw = new PrintWriter(new FileWriter(filename)); + pw = new PrintWriter(filename, "UTF-8"); } catch (IOException e) { throw new RuntimeException(e); } writeHeader(pw); if (!queues.isEmpty()) { - printQueues(pw, queues); + printQueues(pw, queues, useLegacyTagNameForQueues); } if (userSettings != null) { printUserSettings(pw); } - addIfPresent(pw, "queueMaxAppsDefault", - () -> createNumberSupplier(queueMaxAppsDefault)); - addIfPresent(pw, "queueMaxResourcesDefault", - () -> queueMaxResourcesDefault); - addIfPresent(pw, "userMaxAppsDefault", - () -> createNumberSupplier(userMaxAppsDefault)); - addIfPresent(pw, "queueMaxAMShareDefault", - () -> createNumberSupplier(queueMaxAMShareDefault)); + if (queuePlacementPolicy != null) { + printQueuePlacementPolicy(pw); + } + + addIfPresent(pw, "queueMaxAppsDefault", queueMaxAppsDefault); + addIfPresent(pw, "queueMaxResourcesDefault", queueMaxResourcesDefault); + addIfPresent(pw, "userMaxAppsDefault", userMaxAppsDefault); + addIfPresent(pw, "queueMaxAMShareDefault", queueMaxAMShareDefault); addIfPresent(pw, "defaultMinSharePreemptionTimeout", - () -> createNumberSupplier(defaultMinSharePreemptionTimeout)); + defaultMinSharePreemptionTimeout); addIfPresent(pw, "defaultFairSharePreemptionTimeout", - () -> createNumberSupplier(defaultFairSharePreemptionTimeout)); + defaultFairSharePreemptionTimeout); addIfPresent(pw, "defaultFairSharePreemptionThreshold", - () -> createNumberSupplier(defaultFairSharePreemptionThreshold)); + defaultFairSharePreemptionThreshold); addIfPresent(pw, "defaultQueueSchedulingPolicy", - () -> defaultQueueSchedulingPolicy); + defaultQueueSchedulingPolicy); + addIfPresent(pw, "reservation-agent", reservationAgent); + addIfPresent(pw, "reservation-policy", reservationPolicy); + writeFooter(pw); pw.close(); } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/allocationfile/UserSettings.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/allocationfile/UserSettings.java index 7a5656e5f3a..e8457d0e8ec 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/allocationfile/UserSettings.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/allocationfile/UserSettings.java @@ -23,7 +23,7 @@ * Value class that stores user settings and can render data in XML format, * see {@link #render()}. */ -class UserSettings { +public class UserSettings { private final String username; private final Integer maxRunningApps; @@ -36,8 +36,7 @@ public String render() { StringWriter sw = new StringWriter(); PrintWriter pw = new PrintWriter(sw); addStartTag(pw); - AllocationFileWriter.addIfPresent(pw, "maxRunningApps", - () -> AllocationFileWriter.createNumberSupplier(maxRunningApps)); + AllocationFileWriter.addIfPresent(pw, "maxRunningApps", maxRunningApps); addEndTag(pw); pw.close(); @@ -56,12 +55,10 @@ private void addEndTag(PrintWriter pw) { * Builder class for {@link UserSettings} */ public static class Builder { - private final AllocationFileWriter allocationFileWriter; private final String username; private Integer maxRunningApps; - Builder(AllocationFileWriter allocationFileWriter, String username) { - this.allocationFileWriter = allocationFileWriter; + public Builder(String username) { this.username = username; } @@ -70,11 +67,8 @@ public Builder maxRunningApps(int value) { return this; } - public AllocationFileWriter build() { - UserSettings userSettings = new UserSettings(this); - allocationFileWriter.setUserSettings(userSettings); - - return allocationFileWriter; + public UserSettings build() { + return new UserSettings(this); } } } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/converter/FSConfigConverterTestCommons.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/converter/FSConfigConverterTestCommons.java index 0f57241c953..768ee4398b3 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/converter/FSConfigConverterTestCommons.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/converter/FSConfigConverterTestCommons.java @@ -19,6 +19,8 @@ import org.apache.commons.io.FileUtils; import org.apache.commons.lang3.StringUtils; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.FairSchedulerConfiguration; +import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.allocationfile.AllocationFileQueue; +import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.allocationfile.AllocationFileWriter; import java.io.ByteArrayOutputStream; import java.io.File; @@ -113,38 +115,23 @@ public static void setupFSConfigConversionFiles() throws IOException { configureDummyConversionRulesFile(); } - @SuppressWarnings("checkstyle:linelength") - public static void configureFairSchedulerXml() throws IOException { - PrintWriter out = new PrintWriter(new FileWriter(FS_ALLOC_FILE)); - out.println(""); - out.println(""); - out.println("-1.0"); - out.println("fair"); - addQueue(out, ""); - out.println(""); - out.close(); - } - - @SuppressWarnings("checkstyle:linelength") - private static void addQueue(PrintWriter out, String additionalConfig) { - out.println(""); - out.println(" fair"); - out.println(" 1.0"); - out.println(" 100"); - out.println(" 120"); - out.println(" .5"); - - if (StringUtils.isNotEmpty(additionalConfig)) { - out.println(additionalConfig); - } - out.println(""); + public static void configureFairSchedulerXml() { + AllocationFileWriter.create() + .queueMaxAMShareDefault(-1.0f) + .defaultQueueSchedulingPolicy("fair") + .addQueue(new AllocationFileQueue.Builder("root") + .schedulingPolicy("fair") + .weight(1.0f) + .fairSharePreemptionTimeout(100) + .minSharePreemptionTimeout(120) + .fairSharePreemptionThreshold(0.5f) + .build()) + .writeToFile(FS_ALLOC_FILE); } - public static void configureEmptyFairSchedulerXml() throws IOException { - PrintWriter out = new PrintWriter(new FileWriter(FS_ALLOC_FILE)); - out.println(""); - out.println(""); - out.close(); + public static void configureEmptyFairSchedulerXml() { + AllocationFileWriter.create() + .writeToFile(FS_ALLOC_FILE); } public static void configureYarnSiteXmlWithFsAllocFileDefined() diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesAppsModification.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesAppsModification.java index 5d99e0d169d..61dc85ea904 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesAppsModification.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesAppsModification.java @@ -26,9 +26,7 @@ import java.io.ByteArrayInputStream; import java.io.DataInputStream; import java.io.File; -import java.io.FileWriter; import java.io.IOException; -import java.io.PrintWriter; import java.io.StringReader; import java.io.StringWriter; import java.net.URI; @@ -81,6 +79,11 @@ import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacitySchedulerConfiguration; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.FairScheduler; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.FairSchedulerConfiguration; + +import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair + .allocationfile.AllocationFileQueue; +import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair + .allocationfile.AllocationFileWriter; import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.AppPriority; import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.AppQueue; import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.AppState; @@ -202,23 +205,15 @@ public void configureScheduler() { private class FairTestServletModule extends TestServletModule { @Override public void configureScheduler() { - try { - PrintWriter out = new PrintWriter(new FileWriter(FS_ALLOC_FILE)); - out.println(""); - out.println(""); - out.println(""); - out.println(" someuser "); - out.println(" "); - out.println(" someuser "); - out.println(" "); - out.println(" "); - out.println(" someuser "); - out.println(" "); - out.println(""); - out.println(""); - out.close(); - } catch(IOException e) { - } + AllocationFileWriter.create() + .addQueue(new AllocationFileQueue.Builder("root") + .aclAdministerApps("someuser ") + .subQueue(new AllocationFileQueue.Builder("default") + .aclAdministerApps("someuser ").build()) + .subQueue(new AllocationFileQueue.Builder("test") + .aclAdministerApps("someuser ").build()) + .build()) + .writeToFile(FS_ALLOC_FILE); conf.set(FairSchedulerConfiguration.ALLOCATION_FILE, FS_ALLOC_FILE); conf.set(YarnConfiguration.RM_SCHEDULER, FairScheduler.class.getName()); } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesReservation.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesReservation.java index adf6d7cb36c..80145764635 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesReservation.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesReservation.java @@ -24,9 +24,7 @@ import static org.junit.Assert.assertTrue; import java.io.File; -import java.io.FileWriter; import java.io.IOException; -import java.io.PrintWriter; import java.io.StringReader; import java.net.URL; import java.util.Arrays; @@ -53,6 +51,11 @@ import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacitySchedulerConfiguration; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.FairScheduler; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.FairSchedulerConfiguration; + +import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair + .allocationfile.AllocationFileQueue; +import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair + .allocationfile.AllocationFileWriter; import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.ReservationDeleteRequestInfo; import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.ReservationSubmissionRequestInfo; import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.ReservationUpdateRequestInfo; @@ -181,27 +184,17 @@ public void configureScheduler() { private static class FairTestServletModule extends TestServletModule { @Override public void configureScheduler() { - try { - PrintWriter out = new PrintWriter(new FileWriter(FS_ALLOC_FILE)); - out.println(""); - out.println(""); - out.println(""); - out.println(" someuser "); - out.println(" "); - out.println(" someuser "); - out.println(" "); - out.println(" "); - out.println(" "); - out.println(" "); - out.println(" someuser "); - out.println(" "); - out.println(""); - out.println("drf" + - ""); - out.println(""); - out.close(); - } catch (IOException e) { - } + AllocationFileWriter.create() + .defaultQueueSchedulingPolicy("drf") + .addQueue(new AllocationFileQueue.Builder("root") + .aclAdministerApps("someuser ") + .subQueue(new AllocationFileQueue.Builder("default") + .aclAdministerApps("someuser ").build()) + .subQueue(new AllocationFileQueue.Builder("dedicated") + .reservation() + .aclAdministerApps("someuser ").build()) + .build()) + .writeToFile(FS_ALLOC_FILE); conf.set(FairSchedulerConfiguration.ALLOCATION_FILE, FS_ALLOC_FILE); conf.set(YarnConfiguration.RM_SCHEDULER, FairScheduler.class.getName()); } -- 2.21.0