diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/api/impl/TestYarnClientWithReservation.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/api/impl/TestYarnClientWithReservation.java
index 0836b7b9c97..81f6ac06808 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/api/impl/TestYarnClientWithReservation.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/api/impl/TestYarnClientWithReservation.java
@@ -45,6 +45,11 @@
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacitySchedulerConfiguration;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.FairScheduler;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.FairSchedulerConfiguration;
+
+import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair
+ .allocationfile.AllocationFileQueue;
+import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair
+ .allocationfile.AllocationFileWriter;
import org.apache.hadoop.yarn.util.Clock;
import org.apache.hadoop.yarn.util.UTCClock;
import org.junit.Assert;
@@ -134,25 +139,16 @@ private Configuration configureReservationForCapacityScheduler() {
private Configuration configureReservationForFairScheduler() {
Configuration conf = new Configuration();
- try {
- PrintWriter out = new PrintWriter(new FileWriter(FS_ALLOC_FILE));
- out.println("");
- out.println("");
- out.println("");
- out.println(" ");
- out.println(" ");
- out.println(" ");
- // set weight to 10 to make sure this queue get enough steady fair share
- out.println(" 10");
- out.println(" ");
- out.println("");
- out.println("drf" +
- "");
- out.println("");
- out.close();
- } catch (IOException e) {
- Assert.fail(e.getMessage());
- }
+ AllocationFileWriter.create()
+ .defaultQueueSchedulingPolicy("drf")
+ .addQueue(new AllocationFileQueue.Builder("root")
+ .subQueue(new AllocationFileQueue.Builder("default").build())
+ .subQueue(new AllocationFileQueue.Builder("dedicated")
+ .reservation()
+ .weight(10)
+ .build())
+ .build())
+ .writeToFile(FS_ALLOC_FILE);
conf.set(FairSchedulerConfiguration.ALLOCATION_FILE, FS_ALLOC_FILE);
return conf;
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/ParameterizedSchedulerTestBase.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/ParameterizedSchedulerTestBase.java
index 4de16dc88b0..f4193fb6cc0 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/ParameterizedSchedulerTestBase.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/ParameterizedSchedulerTestBase.java
@@ -26,6 +26,12 @@
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.FairScheduler;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.FairSchedulerConfiguration;
+
+
+import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair
+ .allocationfile.AllocationFileQueue;
+import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair
+ .allocationfile.AllocationFileWriter;
import org.junit.After;
import org.junit.runner.RunWith;
import org.junit.runners.Parameterized;
@@ -93,20 +99,17 @@ public ParameterizedSchedulerTestBase(SchedulerType type)
protected void configureFairScheduler(YarnConfiguration conf)
throws IOException {
// Disable queueMaxAMShare limitation for fair scheduler
- PrintWriter out = new PrintWriter(new FileWriter(FS_ALLOC_FILE));
- out.println("");
- out.println("");
- out.println("-1.0");
- out.println("fair");
- out.println("");
- out.println(" drf");
- out.println(" 1.0");
- out.println(" 100");
- out.println(" 120");
- out.println(" .5");
- out.println("");
- out.println("");
- out.close();
+ AllocationFileWriter.create()
+ .defaultQueueSchedulingPolicy("fair")
+ .queueMaxAMShareDefault(-1.0)
+ .addQueue(new AllocationFileQueue.Builder("root")
+ .schedulingPolicy("drf")
+ .weight(1.0f)
+ .fairSharePreemptionTimeout(100)
+ .minSharePreemptionTimeout(120)
+ .fairSharePreemptionThreshold(.5)
+ .build())
+ .writeToFile(FS_ALLOC_FILE);
conf.set(FairSchedulerConfiguration.ALLOCATION_FILE, FS_ALLOC_FILE);
conf.setLong(FairSchedulerConfiguration.UPDATE_INTERVAL_MS, 10);
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/ReservationACLsTestBase.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/ReservationACLsTestBase.java
index c8ee00e60bc..e9ba98ea07f 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/ReservationACLsTestBase.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/ReservationACLsTestBase.java
@@ -53,6 +53,12 @@
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacitySchedulerConfiguration;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.FairScheduler;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.FairSchedulerConfiguration;
+
+
+import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair
+ .allocationfile.AllocationFileQueue;
+import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair
+ .allocationfile.AllocationFileWriter;
import org.apache.hadoop.yarn.server.utils.BuilderUtils;
import org.junit.After;
import org.junit.Assert;
@@ -552,48 +558,34 @@ private static Configuration createCapacitySchedulerConfiguration() {
return csConf;
}
- private static Configuration createFairSchedulerConfiguration() throws
- IOException {
+ private static Configuration createFairSchedulerConfiguration() {
FairSchedulerConfiguration fsConf = new FairSchedulerConfiguration();
final String TEST_DIR = new File(System.getProperty("test.build.data",
"/tmp")).getAbsolutePath();
final String ALLOC_FILE = new File(TEST_DIR, "test-queues.xml")
.getAbsolutePath();
- PrintWriter out = new PrintWriter(new FileWriter(ALLOC_FILE));
- out.println("");
- out.println("");
- out.println(" drf" +
- "");
- out.println(" ");
- out.println(" " +
- "queueA_user,common_user " +
- "");
- out.println(" " +
- "queueA_admin " +
- "");
- out.println(" common_user ");
- out.println(" queueA_user,common_user ");
- out.println(" queueA_admin ");
- out.println(" ");
- out.println(" ");
- out.println(" ");
- out.println(" queueB_user,common_user ");
- out.println(" queueB_admin ");
- out.println(" " +
- "queueB_user,common_user " +
- "");
- out.println(" " +
- "queueB_admin " +
- "");
- out.println(" common_user ");
- out.println(" ");
- out.println(" ");
- out.println(" ");
- out.println(" ");
- out.println(" ");
- out.println("");
- out.close();
+
+ AllocationFileWriter.create()
+ .defaultQueueSchedulingPolicy("drf")
+ .addQueue(new AllocationFileQueue.Builder("queueA")
+ .aclSubmitReservations("queueA_user,common_user ")
+ .aclAdministerReservations("queueA_admin ")
+ .aclListReservations("common_user ")
+ .aclSubmitApps("queueA_user,common_user ")
+ .aclAdministerApps("queueA_admin ")
+ .reservation().build())
+ .addQueue(new AllocationFileQueue.Builder("queueB")
+ .aclSubmitReservations("queueB_user,common_user ")
+ .aclAdministerReservations("queueB_admin ")
+ .aclListReservations("common_user ")
+ .aclSubmitApps("queueB_user,common_user ")
+ .aclAdministerApps("queueB_admin ")
+ .reservation().build())
+ .addQueue(new AllocationFileQueue.Builder("queueC")
+ .reservation().build())
+ .writeToFile(ALLOC_FILE);
+
fsConf.set(FairSchedulerConfiguration.ALLOCATION_FILE, ALLOC_FILE);
fsConf.setBoolean(YarnConfiguration.RM_RESERVATION_SYSTEM_ENABLE, true);
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestApplicationACLs.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestApplicationACLs.java
index c7ed02c9c29..168a30e61c2 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestApplicationACLs.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestApplicationACLs.java
@@ -35,6 +35,16 @@
import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMApp;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.FairScheduler;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.FairSchedulerConfiguration;
+
+
+import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair
+ .allocationfile.AllocationFileQueue;
+import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair
+ .allocationfile.AllocationFileQueuePlacementPolicy;
+import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair
+ .allocationfile.AllocationFileQueuePlacementRule;
+import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair
+ .allocationfile.AllocationFileWriter;
import org.junit.After;
import org.junit.Assert;
import org.apache.commons.logging.Log;
@@ -180,25 +190,22 @@ public void tearDown() {
}
@Override
- protected void configureFairScheduler(YarnConfiguration conf)
- throws IOException {
+ protected void configureFairScheduler(YarnConfiguration conf) {
final String testDir = new File(System.getProperty("test.build.data",
"/tmp")).getAbsolutePath();
final String allocFile = new File(testDir, "test-queues.xml")
.getAbsolutePath();
- PrintWriter out = new PrintWriter(new FileWriter(allocFile));
- out.println("");
- out.println("");
- out.println("");
- out.println(" ");
- out.println(" ");
- out.println("");
- out.println("");
- out.println(" ");
- out.println(" ");
- out.println("");
- out.println("");
- out.close();
+
+ AllocationFileWriter.create()
+ .addQueue(new AllocationFileQueue.Builder("root")
+ .subQueue(new AllocationFileQueue.Builder("default").build())
+ .build())
+ .queuePlacementPolicy(new AllocationFileQueuePlacementPolicy()
+ .addRule(new AllocationFileQueuePlacementRule("specified")
+ .create(false))
+ .addRule(new AllocationFileQueuePlacementRule("reject")))
+ .writeToFile(allocFile);
+
conf.set(FairSchedulerConfiguration.ALLOCATION_FILE, allocFile);
}
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestApplicationMasterService.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestApplicationMasterService.java
index 562ba5d5062..98292088cfd 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestApplicationMasterService.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestApplicationMasterService.java
@@ -92,6 +92,7 @@
import org.junit.Before;
import org.junit.Test;
+
public class TestApplicationMasterService {
private static final Log LOG = LogFactory
.getLog(TestApplicationMasterService.class);
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/reservation/ReservationSystemTestUtil.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/reservation/ReservationSystemTestUtil.java
index eef86a44990..96889cf8c61 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/reservation/ReservationSystemTestUtil.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/reservation/ReservationSystemTestUtil.java
@@ -56,6 +56,8 @@
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacitySchedulerConfiguration;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.event.NodeAddedSchedulerEvent;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.FairScheduler;
+import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.allocationfile.AllocationFileQueue;
+import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.allocationfile.AllocationFileWriter;
import org.apache.hadoop.yarn.server.resourcemanager.security.ClientToAMTokenSecretManagerInRM;
import org.apache.hadoop.yarn.server.resourcemanager.security.NMTokenSecretManagerInRM;
import org.apache.hadoop.yarn.server.resourcemanager.security.RMContainerTokenSecretManager;
@@ -103,62 +105,46 @@ public static void validateReservationQueue(
.assertTrue(plan.getSharingPolicy() instanceof CapacityOverTimePolicy);
}
- public static void setupFSAllocationFile(String allocationFile)
- throws IOException {
- PrintWriter out = new PrintWriter(new FileWriter(allocationFile));
- out.println("");
- out.println("");
- out.println("");
- out.println("1");
- out.println("");
- out.println("");
- out.println("1");
- out.println("");
- out.println("3");
- out.println("");
- out.println("");
- out.println("7");
- out.println("");
- out.println("");
- out.println("");
- out.println("");
- out.println("8");
- out.println("");
- out.println(
- "drf");
- out.println("");
- out.close();
+ public static void setupFSAllocationFile(String allocationFile) {
+ AllocationFileWriter.create()
+ .defaultQueueSchedulingPolicy("drf")
+ .addQueue(new AllocationFileQueue.Builder("default")
+ .weight(1).build())
+ .addQueue(new AllocationFileQueue.Builder("a")
+ .weight(1)
+ .subQueue(new AllocationFileQueue.Builder("a1")
+ .weight(3).build())
+ .subQueue(new AllocationFileQueue.Builder("a2")
+ .weight(7).build())
+ .build())
+ .addQueue(new AllocationFileQueue.Builder("dedicated")
+ .weight(8)
+ .reservation()
+ .build())
+ .writeToFile(allocationFile);
}
- public static void updateFSAllocationFile(String allocationFile)
- throws IOException {
- PrintWriter out = new PrintWriter(new FileWriter(allocationFile));
- out.println("");
- out.println("");
- out.println("");
- out.println("5");
- out.println("");
- out.println("");
- out.println("5");
- out.println("");
- out.println("3");
- out.println("");
- out.println("");
- out.println("7");
- out.println("");
- out.println("");
- out.println("");
- out.println("");
- out.println("10");
- out.println("");
- out.println("");
- out.println("");
- out.println("80");
- out.println("");
- out.println(
- "drf");
- out.println("");
- out.close();
+ public static void updateFSAllocationFile(String allocationFile) {
+ AllocationFileWriter.create()
+ .defaultQueueSchedulingPolicy("drf")
+ .addQueue(new AllocationFileQueue.Builder("default")
+ .weight(5).build())
+ .addQueue(new AllocationFileQueue.Builder("a")
+ .weight(5)
+ .subQueue(new AllocationFileQueue.Builder("a1")
+ .weight(3).build())
+ .subQueue(new AllocationFileQueue.Builder("a2")
+ .weight(7).build())
+ .build())
+ .addQueue(new AllocationFileQueue.Builder("dedicated")
+ .weight(10)
+ .reservation()
+ .build())
+ .addQueue(new AllocationFileQueue.Builder("reservation")
+ .weight(80)
+ .reservation()
+ .build())
+ .writeToFile(allocationFile);
}
public static FairScheduler setupFairScheduler(RMContext rmContext,
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestAllocationFileLoaderService.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestAllocationFileLoaderService.java
index 4a7461da79b..3ebe215a1f2 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestAllocationFileLoaderService.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestAllocationFileLoaderService.java
@@ -27,17 +27,21 @@
import org.apache.hadoop.yarn.conf.YarnConfiguration;
import org.apache.hadoop.yarn.server.resourcemanager.reservation.ReservationSchedulerConfiguration;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.QueuePlacementRule.NestedUserQueue;
+import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.allocationfile.AllocationFileQueue;
+import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.allocationfile.AllocationFileQueuePlacementPolicy;
+import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.allocationfile.AllocationFileQueuePlacementRule;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.allocationfile.AllocationFileWriter;
+import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.allocationfile.UserSettings;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.policies.DominantResourceFairnessPolicy;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.policies.FairSharePolicy;
import org.apache.hadoop.yarn.util.ControlledClock;
import org.apache.hadoop.yarn.util.resource.Resources;
+import org.junit.After;
import org.junit.Test;
+
import java.io.File;
-import java.io.FileOutputStream;
import java.io.FileWriter;
import java.io.IOException;
-import java.io.OutputStreamWriter;
import java.io.PrintWriter;
import java.net.URISyntaxException;
import java.net.URL;
@@ -58,6 +62,11 @@
"test-queues").getAbsolutePath();
private static final String TEST_FAIRSCHED_XML = "test-fair-scheduler.xml";
+ @After
+ public void teardown() {
+ new File(ALLOC_FILE).delete();
+ }
+
@Test
public void testGetAllocationFileFromFileSystem()
throws IOException, URISyntaxException {
@@ -114,18 +123,13 @@ public void testGetAllocationFileFromClasspath() {
@Test (timeout = 10000)
public void testReload() throws Exception {
- PrintWriter out = new PrintWriter(new FileWriter(ALLOC_FILE));
- out.println("");
- out.println("");
- out.println(" ");
- out.println(" 1");
- out.println(" ");
- out.println(" ");
- out.println(" ");
- out.println(" ");
- out.println(" ");
- out.println("");
- out.close();
+ AllocationFileWriter.create()
+ .addQueue(new AllocationFileQueue.Builder("queueA")
+ .maxRunningApps(1).build())
+ .addQueue(new AllocationFileQueue.Builder("queueB").build())
+ .queuePlacementPolicy(new AllocationFileQueuePlacementPolicy()
+ .addRule(new AllocationFileQueuePlacementRule("default")))
+ .writeToFile(ALLOC_FILE);
ControlledClock clock = new ControlledClock();
clock.setTime(0);
@@ -157,21 +161,15 @@ public void testReload() throws Exception {
confHolder.allocConf = null;
// Modify file and advance the clock
- out = new PrintWriter(new FileWriter(ALLOC_FILE));
- out.println("");
- out.println("");
- out.println(" ");
- out.println(" 3");
- out.println(" ");
- out.println(" ");
- out.println(" ");
- out.println(" ");
- out.println(" ");
- out.println(" ");
- out.println(" ");
- out.println(" ");
- out.println("");
- out.close();
+ AllocationFileWriter.create()
+ .addQueue(new AllocationFileQueue.Builder("queueB")
+ .maxRunningApps(3).build())
+ .queuePlacementPolicy(new AllocationFileQueuePlacementPolicy()
+ .addRule(new AllocationFileQueuePlacementRule("specified"))
+ .addRule(new AllocationFileQueuePlacementRule("nestedUserQueue")
+ .addNestedRule(new AllocationFileQueuePlacementRule("primaryGroup")))
+ .addRule(new AllocationFileQueuePlacementRule("default")))
+ .writeToFile(ALLOC_FILE);
clock.tickMsec(System.currentTimeMillis()
+ AllocationFileLoaderService.ALLOC_RELOAD_WAIT_MS + 10000);
@@ -205,54 +203,54 @@ public void testAllocationFileParsing() throws Exception {
conf.set(FairSchedulerConfiguration.ALLOCATION_FILE, ALLOC_FILE);
AllocationFileLoaderService allocLoader = new AllocationFileLoaderService();
- AllocationFileWriter
- .create()
+ AllocationFileWriter.create()
// Give queue A a minimum of 1024 M
- .queue("queueA")
- .minResources("1024mb,0vcores")
- .maxResources("2048mb,10vcores")
- .buildQueue()
+ .addQueue(new AllocationFileQueue.Builder("queueA")
+ .minResources("1024mb,0vcores")
+ .maxResources("2048mb,10vcores")
+ .build())
// Give queue B a minimum of 2048 M
- .queue("queueB")
+ .addQueue(new AllocationFileQueue.Builder("queueB")
.minResources("2048mb,0vcores")
.maxResources("5120mb,110vcores")
.aclAdministerApps("alice,bob admins")
.schedulingPolicy("fair")
- .buildQueue()
+ .build())
// Give queue C no minimum
- .queue("queueC")
- .minResources("5120mb,0vcores")
- .aclSubmitApps("alice,bob admins")
- .buildQueue()
+ .addQueue(new AllocationFileQueue.Builder("queueC")
+ .minResources("5120mb,0vcores")
+ .aclSubmitApps("alice,bob admins")
+ .build())
// Give queue D a limit of 3 running apps and 0.4f maxAMShare
- .queue("queueD")
- .maxRunningApps(3)
- .maxAMShare(0.4)
- .buildQueue()
+ .addQueue(new AllocationFileQueue.Builder("queueD")
+ .maxRunningApps(3)
+ .maxAMShare(0.4)
+ .build())
// Give queue E a preemption timeout of one minute
- .queue("queueE")
- .minSharePreemptionTimeout(60)
- .buildQueue()
+ .addQueue(new AllocationFileQueue.Builder("queueE")
+ .minSharePreemptionTimeout(60)
+ .build())
// Make queue F a parent queue without configured leaf queues
// using the 'type' attribute
- .queue("queueF")
- .parent(true)
- .maxChildResources("2048mb,64vcores")
- .buildQueue()
- .queue("queueG")
- .maxChildResources("2048mb,64vcores")
- .fairSharePreemptionTimeout(120)
- .minSharePreemptionTimeout(50)
- .fairSharePreemptionThreshold(0.6)
+ .addQueue(new AllocationFileQueue.Builder("queueF")
+ .parent(true)
+ .maxChildResources("2048mb,64vcores")
+ .build())
+ .addQueue(new AllocationFileQueue.Builder("queueG")
+ .maxChildResources("2048mb,64vcores")
+ .fairSharePreemptionTimeout(120)
+ .minSharePreemptionTimeout(50)
+ .fairSharePreemptionThreshold(0.6)
// Create hierarchical queues G,H, with different min/fair
// share preemption timeouts and preemption thresholds.
// Also add a child default to make sure it doesn't impact queue H.
- .subQueue("queueH")
- .fairSharePreemptionTimeout(180)
- .minSharePreemptionTimeout(40)
- .fairSharePreemptionThreshold(0.7)
- .buildSubQueue()
- .buildQueue()
+ .subQueue(new AllocationFileQueue.Builder("queueH")
+ .fairSharePreemptionTimeout(180)
+ .minSharePreemptionTimeout(40)
+ .fairSharePreemptionThreshold(0.7)
+ .maxContainerAllocation("1024mb,8vcores")
+ .build())
+ .build())
// Set default limit of apps per queue to 15
.queueMaxAppsDefault(15)
// Set default limit of max resource per queue to 4G and 100 cores
@@ -270,9 +268,9 @@ public void testAllocationFileParsing() throws Exception {
// Set default scheduling policy to DRF
.defaultQueueSchedulingPolicy("drf")
// Give user1 a limit of 10 jobs
- .userSettings("user1")
+ .userSettings(new UserSettings.Builder("user1")
.maxRunningApps(10)
- .build()
+ .build())
.writeToFile(ALLOC_FILE);
allocLoader.init(conf);
@@ -447,48 +445,51 @@ public void testBackwardsCompatibleAllocationFileParsing() throws Exception {
conf.set(FairSchedulerConfiguration.ALLOCATION_FILE, ALLOC_FILE);
AllocationFileLoaderService allocLoader = new AllocationFileLoaderService();
- PrintWriter out = new PrintWriter(new FileWriter(ALLOC_FILE));
- out.println("");
- out.println("");
- // Give queue A a minimum of 1024 M
- out.println("");
- out.println("1024mb,0vcores");
- out.println("");
- // Give queue B a minimum of 2048 M
- out.println("");
- out.println("2048mb,0vcores");
- out.println("alice,bob admins");
- out.println("");
- // Give queue C no minimum
- out.println("");
- out.println("alice,bob admins");
- out.println("");
- // Give queue D a limit of 3 running apps
- out.println("");
- out.println("3");
- out.println("");
- // Give queue E a preemption timeout of one minute and 0.3f threshold
- out.println("");
- out.println("60");
- out.println("0.3");
- out.println("");
- // Set default limit of apps per queue to 15
- out.println("15");
- // Set default limit of apps per user to 5
- out.println("5");
- // Give user1 a limit of 10 jobs
- out.println("");
- out.println("10");
- out.println("");
- // Set default min share preemption timeout to 2 minutes
- out.println("120"
- + "");
- // Set fair share preemption timeout to 5 minutes
- out.println("300");
- // Set default fair share preemption threshold to 0.6f
- out.println("0.6");
- out.println("");
- out.close();
+ AllocationFileWriter.create()
+ .useLegacyTagNameForQueues()
+ // Give queue A a minimum of 1024 M
+ .addQueue(new AllocationFileQueue.Builder("queueA")
+ .minResources("1024mb,0vcores")
+ .build())
+ // Give queue B a minimum of 2048 M
+ .addQueue(new AllocationFileQueue.Builder("queueB")
+ .minResources("2048mb,0vcores")
+ .aclAdministerApps("alice,bob admins")
+ .build())
+ // Give queue C no minimum
+ .addQueue(new AllocationFileQueue.Builder("queueC")
+ .aclSubmitApps("alice,bob admins")
+ .build())
+ // Give queue D a limit of 3 running apps
+ .addQueue(new AllocationFileQueue.Builder("queueD")
+ .maxRunningApps(3)
+ .build())
+ // Give queue E a preemption timeout of one minute and 0.3f threshold
+ .addQueue(new AllocationFileQueue.Builder("queueE")
+ .minSharePreemptionTimeout(60)
+ .fairSharePreemptionThreshold(0.3)
+ .build())
+ // Set default limit of apps per queue to 15
+ .queueMaxAppsDefault(15)
+ // Set default limit of apps per user to 5
+ .userMaxAppsDefault(5)
+ // Set default limit of max resource per queue to 4G and 100 cores
+ .queueMaxResourcesDefault("4096mb,100vcores")
+ // Set default limit of AMResourceShare to 0.5f
+ .queueMaxAMShareDefault(0.5)
+ // Set default min share preemption timeout to 2 minutes
+ .defaultMinSharePreemptionTimeout(120)
+ // Set default fair share preemption timeout to 5 minutes
+ .defaultFairSharePreemptionTimeout(300)
+ // Set default fair share preemption threshold to 0.6
+ .defaultFairSharePreemptionThreshold(0.6)
+ // Set default scheduling policy to DRF
+ .defaultQueueSchedulingPolicy("drf")
+ // Give user1 a limit of 10 jobs
+ .userSettings(new UserSettings.Builder("user1")
+ .maxRunningApps(10)
+ .build())
+ .writeToFile(ALLOC_FILE);
allocLoader.init(conf);
ReloadListener confHolder = new ReloadListener();
@@ -576,11 +577,7 @@ public void testSimplePlacementPolicyFromConf() throws Exception {
conf.setBoolean(FairSchedulerConfiguration.ALLOW_UNDECLARED_POOLS, false);
conf.setBoolean(FairSchedulerConfiguration.USER_AS_DEFAULT_QUEUE, false);
- PrintWriter out = new PrintWriter(new FileWriter(ALLOC_FILE));
- out.println("");
- out.println("");
- out.println("");
- out.close();
+ AllocationFileWriter.create().writeToFile(ALLOC_FILE);
AllocationFileLoaderService allocLoader = new AllocationFileLoaderService();
allocLoader.init(conf);
@@ -606,15 +603,10 @@ public void testQueueAlongsideRoot() throws Exception {
Configuration conf = new Configuration();
conf.set(FairSchedulerConfiguration.ALLOCATION_FILE, ALLOC_FILE);
- PrintWriter out = new PrintWriter(new FileWriter(ALLOC_FILE));
- out.println("");
- out.println("");
- out.println("");
- out.println("");
- out.println("");
- out.println("");
- out.println("");
- out.close();
+ AllocationFileWriter.create()
+ .addQueue(new AllocationFileQueue.Builder("root").build())
+ .addQueue(new AllocationFileQueue.Builder("other").build())
+ .writeToFile(ALLOC_FILE);
AllocationFileLoaderService allocLoader = new AllocationFileLoaderService();
allocLoader.init(conf);
@@ -632,13 +624,9 @@ public void testQueueNameContainingPeriods() throws Exception {
Configuration conf = new Configuration();
conf.set(FairSchedulerConfiguration.ALLOCATION_FILE, ALLOC_FILE);
- PrintWriter out = new PrintWriter(new FileWriter(ALLOC_FILE));
- out.println("");
- out.println("");
- out.println("");
- out.println("");
- out.println("");
- out.close();
+ AllocationFileWriter.create()
+ .addQueue(new AllocationFileQueue.Builder("parent1.child").build())
+ .writeToFile(ALLOC_FILE);
AllocationFileLoaderService allocLoader = new AllocationFileLoaderService();
allocLoader.init(conf);
@@ -656,13 +644,9 @@ public void testQueueNameContainingOnlyWhitespace() throws Exception {
Configuration conf = new Configuration();
conf.set(FairSchedulerConfiguration.ALLOCATION_FILE, ALLOC_FILE);
- PrintWriter out = new PrintWriter(new FileWriter(ALLOC_FILE));
- out.println("");
- out.println("");
- out.println("");
- out.println("");
- out.println("");
- out.close();
+ AllocationFileWriter.create()
+ .addQueue(new AllocationFileQueue.Builder(" ").build())
+ .writeToFile(ALLOC_FILE);
AllocationFileLoaderService allocLoader = new AllocationFileLoaderService();
allocLoader.init(conf);
@@ -676,15 +660,12 @@ public void testParentTagWithReservation() throws Exception {
Configuration conf = new Configuration();
conf.set(FairSchedulerConfiguration.ALLOCATION_FILE, ALLOC_FILE);
- PrintWriter out = new PrintWriter(new FileWriter(ALLOC_FILE));
- out.println("");
- out.println("");
- out.println("");
- out.println("");
- out.println("");
- out.println("");
- out.println("");
- out.close();
+ AllocationFileWriter.create()
+ .addQueue(new AllocationFileQueue.Builder("parent")
+ .parent(true)
+ .reservation()
+ .build())
+ .writeToFile(ALLOC_FILE);
AllocationFileLoaderService allocLoader = new AllocationFileLoaderService();
allocLoader.init(conf);
@@ -705,17 +686,13 @@ public void testParentWithReservation() throws Exception {
Configuration conf = new Configuration();
conf.set(FairSchedulerConfiguration.ALLOCATION_FILE, ALLOC_FILE);
- PrintWriter out = new PrintWriter(new FileWriter(ALLOC_FILE));
- out.println("");
- out.println("");
- out.println("");
- out.println("");
- out.println("");
- out.println(" ");
- out.println(" ");
- out.println("");
- out.println("");
- out.close();
+ AllocationFileWriter.create()
+ .addQueue(new AllocationFileQueue.Builder("parent")
+ .parent(true)
+ .subQueue(new AllocationFileQueue.Builder("child").build())
+ .reservation()
+ .build())
+ .writeToFile(ALLOC_FILE);
AllocationFileLoaderService allocLoader = new AllocationFileLoaderService();
allocLoader.init(conf);
@@ -736,15 +713,12 @@ public void testParentTagWithChild() throws Exception {
Configuration conf = new Configuration();
conf.set(FairSchedulerConfiguration.ALLOCATION_FILE, ALLOC_FILE);
- PrintWriter out = new PrintWriter(new FileWriter(ALLOC_FILE));
- out.println("");
- out.println("");
- out.println("");
- out.println(" ");
- out.println(" ");
- out.println("");
- out.println("");
- out.close();
+ AllocationFileWriter.create()
+ .addQueue(new AllocationFileQueue.Builder("parent")
+ .parent(true)
+ .subQueue(new AllocationFileQueue.Builder("child").build())
+ .build())
+ .writeToFile(ALLOC_FILE);
AllocationFileLoaderService allocLoader = new AllocationFileLoaderService();
allocLoader.init(conf);
@@ -768,14 +742,9 @@ public void testQueueNameContainingNBWhitespace() throws Exception {
Configuration conf = new Configuration();
conf.set(FairSchedulerConfiguration.ALLOCATION_FILE, ALLOC_FILE);
- PrintWriter out = new PrintWriter(new OutputStreamWriter(
- new FileOutputStream(ALLOC_FILE), StandardCharsets.UTF_8));
- out.println("");
- out.println("");
- out.println("");
- out.println("");
- out.println("");
- out.close();
+ AllocationFileWriter.create()
+ .addQueue(new AllocationFileQueue.Builder("\u00a0").build())
+ .writeToFile(ALLOC_FILE);
AllocationFileLoaderService allocLoader = new AllocationFileLoaderService();
allocLoader.init(conf);
@@ -792,12 +761,9 @@ public void testDefaultQueueSchedulingModeIsFIFO() throws Exception {
Configuration conf = new Configuration();
conf.set(FairSchedulerConfiguration.ALLOCATION_FILE, ALLOC_FILE);
- PrintWriter out = new PrintWriter(new FileWriter(ALLOC_FILE));
- out.println("");
- out.println("");
- out.println("fifo");
- out.println("");
- out.close();
+ AllocationFileWriter.create()
+ .defaultQueueSchedulingPolicy("fifo")
+ .writeToFile(ALLOC_FILE);
AllocationFileLoaderService allocLoader = new AllocationFileLoaderService();
allocLoader.init(conf);
@@ -811,19 +777,14 @@ public void testReservableQueue() throws Exception {
Configuration conf = new Configuration();
conf.set(FairSchedulerConfiguration.ALLOCATION_FILE, ALLOC_FILE);
- PrintWriter out = new PrintWriter(new FileWriter(ALLOC_FILE));
- out.println("");
- out.println("");
- out.println("");
- out.println("");
- out.println("");
- out.println("");
- out.println("");
- out.println("");
- out.println("DummyAgentName");
- out.println("AnyAdmissionPolicy");
- out.println("");
- out.close();
+ AllocationFileWriter.create()
+ .addQueue(new AllocationFileQueue.Builder("reservable")
+ .reservation()
+ .build())
+ .addQueue(new AllocationFileQueue.Builder("other").build())
+ .reservationAgent("DummyAgentName")
+ .reservationPolicy("AnyAdmissionPolicy")
+ .writeToFile(ALLOC_FILE);
AllocationFileLoaderService allocLoader = new AllocationFileLoaderService();
allocLoader.init(conf);
@@ -868,15 +829,12 @@ public void testReservableCannotBeCombinedWithDynamicUserQueue()
Configuration conf = new Configuration();
conf.set(FairSchedulerConfiguration.ALLOCATION_FILE, ALLOC_FILE);
- PrintWriter out = new PrintWriter(new FileWriter(ALLOC_FILE));
- out.println("");
- out.println("");
- out.println("");
- out.println("");
- out.println("");
- out.println("");
- out.println("");
- out.close();
+ AllocationFileWriter.create()
+ .addQueue(new AllocationFileQueue.Builder("notboth")
+ .parent(true)
+ .reservation()
+ .build())
+ .writeToFile(ALLOC_FILE);
AllocationFileLoaderService allocLoader = new AllocationFileLoaderService();
allocLoader.init(conf);
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestAppRunnability.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestAppRunnability.java
index f5819357ba0..24350838e40 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestAppRunnability.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestAppRunnability.java
@@ -42,6 +42,12 @@
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.event.AppAttemptRemovedSchedulerEvent;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.event.NodeAddedSchedulerEvent;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.event.NodeUpdateSchedulerEvent;
+
+
+import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair
+ .allocationfile.AllocationFileQueue;
+import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair
+ .allocationfile.AllocationFileWriter;
import org.apache.hadoop.yarn.util.resource.Resources;
import org.junit.After;
import org.junit.Before;
@@ -161,17 +167,14 @@ public void testPreemptionVariablesForQueueCreatedRuntime() throws Exception {
}
@Test
- public void testDontAllowUndeclaredPools() throws Exception {
+ public void testDontAllowUndeclaredPools() {
conf.setBoolean(FairSchedulerConfiguration.ALLOW_UNDECLARED_POOLS, false);
conf.set(FairSchedulerConfiguration.ALLOCATION_FILE, ALLOC_FILE);
- PrintWriter out = new PrintWriter(new FileWriter(ALLOC_FILE));
- out.println("");
- out.println("");
- out.println("");
- out.println("");
- out.println("");
- out.close();
+ AllocationFileWriter.create()
+ .addQueue(new AllocationFileQueue.Builder("jerry").build())
+ .writeToFile(ALLOC_FILE);
+
// Restarting resource manager since the file location and content is
// changed.
resourceManager.stop();
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFSAppStarvation.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFSAppStarvation.java
index 9665f9a7bb0..7304d68fbeb 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFSAppStarvation.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFSAppStarvation.java
@@ -22,6 +22,11 @@
import org.apache.hadoop.yarn.server.resourcemanager.MockRM;
import org.apache.hadoop.yarn.server.resourcemanager.rmnode.RMNode;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.event.NodeUpdateSchedulerEvent;
+
+import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair
+ .allocationfile.AllocationFileQueue;
+import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair
+ .allocationfile.AllocationFileWriter;
import org.apache.hadoop.yarn.util.ControlledClock;
import org.junit.After;
@@ -187,52 +192,40 @@ private void setupClusterAndSubmitJobs() throws Exception {
* 4. Submit an app that uses up all resources on the cluster
*/
private void setupStarvedCluster() throws IOException {
- PrintWriter out = new PrintWriter(new FileWriter(ALLOC_FILE));
- out.println("");
- out.println("");
-
- // Default queue
- out.println("");
- out.println("");
-
- // Queue with preemption disabled
- out.println("");
- out.println("0" +
- "");
- out.println("");
-
- // Queue with minshare preemption enabled
- out.println("");
- out.println("0" +
- "");
- out.println("0" +
- "");
- out.println("2048mb,2vcores");
- out.println("");
-
- // FAIR queue with fairshare preemption enabled
- out.println("");
- out.println("1" +
- "");
- out.println("0" +
- "");
- out.println("fair");
- addChildQueue(out, "fair");
- out.println("");
-
- // DRF queue with fairshare preemption enabled
- out.println("");
- out.println("1" +
- "");
- out.println("0" +
- "");
- out.println("drf");
- addChildQueue(out, "drf");
- out.println("");
- out.println("drf" +
- "");
- out.println("");
- out.close();
+ AllocationFileWriter.create()
+ .defaultQueueSchedulingPolicy("drf")
+ // Default queue
+ .addQueue(new AllocationFileQueue.Builder("default").build())
+ // Queue with preemption disabled
+ .addQueue(new AllocationFileQueue.Builder("no-preemption")
+ .fairSharePreemptionThreshold(0).build())
+ // Queue with minshare preemption enabled
+ .addQueue(new AllocationFileQueue.Builder("minshare")
+ .fairSharePreemptionThreshold(0)
+ .minSharePreemptionTimeout(0)
+ .minResources("2048mb,2vcores")
+ .build())
+ // FAIR queue with fairshare preemption enabled
+ .addQueue(new AllocationFileQueue.Builder("fairshare")
+ .fairSharePreemptionThreshold(1)
+ .fairSharePreemptionTimeout(0)
+ .schedulingPolicy("fair")
+ .subQueue(new AllocationFileQueue.Builder("child")
+ .fairSharePreemptionThreshold(1)
+ .fairSharePreemptionTimeout(0)
+ .schedulingPolicy("fair").build())
+ .build())
+ // DRF queue with fairshare preemption enabled
+ .addQueue(new AllocationFileQueue.Builder("drf")
+ .fairSharePreemptionThreshold(1)
+ .fairSharePreemptionTimeout(0)
+ .schedulingPolicy("drf")
+ .subQueue(new AllocationFileQueue.Builder("child")
+ .fairSharePreemptionThreshold(1)
+ .fairSharePreemptionTimeout(0)
+ .schedulingPolicy("drf").build())
+ .build())
+ .writeToFile(ALLOC_FILE.getAbsolutePath());
assertTrue("Allocation file does not exist, not running the test",
ALLOC_FILE.exists());
@@ -258,17 +251,6 @@ private void setupStarvedCluster() throws IOException {
assertEquals(8, scheduler.getSchedulerApp(app).getLiveContainers().size());
}
- private void addChildQueue(PrintWriter out, String policy) {
- // Child queue under fairshare with same settings
- out.println("");
- out.println("1" +
- "");
- out.println("0" +
- "");
- out.println("" + policy + "");
- out.println("");
- }
-
private void submitAppsToEachLeafQueue() {
for (String queue : QUEUES) {
createSchedulingRequest(1024, 1, "root." + queue, "user", 1);
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFSLeafQueue.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFSLeafQueue.java
index 4a738ca07fb..ad2375193c5 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFSLeafQueue.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFSLeafQueue.java
@@ -45,12 +45,28 @@
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.ResourceScheduler;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.event.NodeAddedSchedulerEvent;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.event.NodeUpdateSchedulerEvent;
+import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.allocationfile.AllocationFileQueue;
+import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.allocationfile.AllocationFileWriter;
import org.apache.hadoop.yarn.util.resource.Resources;
import org.junit.After;
import org.junit.Before;
import org.junit.Test;
import org.mockito.Mockito;
+import java.io.File;
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.Collections;
+import java.util.List;
+import java.util.concurrent.CountDownLatch;
+import java.util.concurrent.ExecutorService;
+import java.util.concurrent.TimeUnit;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertTrue;
+import static org.mockito.Mockito.mock;
+
public class TestFSLeafQueue extends FairSchedulerTestBase {
private final static String ALLOC_FILE = new File(TEST_DIR,
TestFSLeafQueue.class.getName() + ".xml").getAbsolutePath();
@@ -100,15 +116,12 @@ public void testUpdateDemand() {
}
@Test (timeout = 5000)
- public void test() throws Exception {
+ public void test() {
conf.set(FairSchedulerConfiguration.ALLOCATION_FILE, ALLOC_FILE);
- PrintWriter out = new PrintWriter(new FileWriter(ALLOC_FILE));
- out.println("");
- out.println("");
- out.println("");
- out.println("");
- out.println("");
- out.close();
+ AllocationFileWriter.create()
+ .addQueue(new AllocationFileQueue.Builder("queueA").build())
+ .addQueue(new AllocationFileQueue.Builder("queueB").build())
+ .writeToFile(ALLOC_FILE);
resourceManager = new MockRM(conf);
resourceManager.start();
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFairScheduler.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFairScheduler.java
index d8da2da5e5d..4f38c0aaa3b 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFairScheduler.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFairScheduler.java
@@ -109,6 +109,11 @@
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.event.NodeAddedSchedulerEvent;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.event.NodeRemovedSchedulerEvent;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.event.NodeUpdateSchedulerEvent;
+import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.allocationfile.AllocationFileQueue;
+import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.allocationfile.AllocationFileQueuePlacementPolicy;
+import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.allocationfile.AllocationFileQueuePlacementRule;
+import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.allocationfile.AllocationFileWriter;
+import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.allocationfile.UserSettings;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.QueuePlacementRule.Default;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.policies.DominantResourceFairnessPolicy;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.policies.FifoPolicy;
@@ -345,19 +350,17 @@ public void testFairShareWithHighMaxResources() throws IOException {
// set queueA and queueB maxResources,
// the sum of queueA and queueB maxResources is more than
// Integer.MAX_VALUE.
- PrintWriter out = new PrintWriter(new FileWriter(ALLOC_FILE));
- out.println("");
- out.println("");
- out.println("");
- out.println("1073741824 mb 1000 vcores");
- out.println(".25");
- out.println("");
- out.println("");
- out.println("1073741824 mb 1000 vcores");
- out.println(".75");
- out.println("");
- out.println("");
- out.close();
+
+ AllocationFileWriter.create()
+ .addQueue(new AllocationFileQueue.Builder("queueA")
+ .maxResources("1073741824 mb 1000 vcores")
+ .weight(.25f)
+ .build())
+ .addQueue(new AllocationFileQueue.Builder("queueB")
+ .maxResources("1073741824 mb 1000 vcores")
+ .weight(.75f)
+ .build())
+ .writeToFile(ALLOC_FILE);
scheduler.init(conf);
scheduler.start();
@@ -399,18 +402,16 @@ public void testFairShareWithHighMaxResources() throws IOException {
public void testFairShareWithLowMaxResources() throws IOException {
PrintWriter out = new PrintWriter(new FileWriter(ALLOC_FILE));
- out.println("");
- out.println("");
- out.println(" ");
- out.println(" 1024 mb 1 vcores");
- out.println(" 0.75");
- out.println(" ");
- out.println(" ");
- out.println(" 3072 mb 3 vcores");
- out.println(" 0.25");
- out.println(" ");
- out.println("");
- out.close();
+ AllocationFileWriter.create()
+ .addQueue(new AllocationFileQueue.Builder("queueA")
+ .maxResources("1024 mb 1 vcores")
+ .weight(.75f)
+ .build())
+ .addQueue(new AllocationFileQueue.Builder("queueB")
+ .maxResources("3072 mb 3 vcores")
+ .weight(.25f)
+ .build())
+ .writeToFile(ALLOC_FILE);
conf.set(FairSchedulerConfiguration.ALLOCATION_FILE, ALLOC_FILE);
scheduler.init(conf);
@@ -469,15 +470,12 @@ public void testFairShareWithLowMaxResources() throws IOException {
*/
@Test
public void testChildMaxResources() throws IOException {
- PrintWriter out = new PrintWriter(new FileWriter(ALLOC_FILE));
-
- out.println("");
- out.println("");
- out.println(" ");
- out.println(" 2048mb,2vcores");
- out.println(" ");
- out.println("");
- out.close();
+ AllocationFileWriter.create()
+ .addQueue(new AllocationFileQueue.Builder("queueA")
+ .parent(true)
+ .maxChildResources("2048mb,2vcores")
+ .build())
+ .writeToFile(ALLOC_FILE);
conf.set(FairSchedulerConfiguration.ALLOCATION_FILE, ALLOC_FILE);
scheduler.init(conf);
@@ -516,14 +514,12 @@ public void testChildMaxResources() throws IOException {
assertEquals("App 2 is not running with the correct number of containers",
2, scheduler.getSchedulerApp(attId2).getLiveContainers().size());
- out = new PrintWriter(new FileWriter(ALLOC_FILE));
- out.println("");
- out.println("");
- out.println(" ");
- out.println(" 3072mb,3vcores");
- out.println(" ");
- out.println("");
- out.close();
+ AllocationFileWriter.create()
+ .addQueue(new AllocationFileQueue.Builder("queueA")
+ .parent(true)
+ .maxChildResources("3072mb,3vcores")
+ .build())
+ .writeToFile(ALLOC_FILE);
scheduler.reinitialize(conf, resourceManager.getRMContext());
@@ -539,14 +535,12 @@ public void testChildMaxResources() throws IOException {
assertEquals("App 2 is not running with the correct number of containers",
3, scheduler.getSchedulerApp(attId2).getLiveContainers().size());
- out = new PrintWriter(new FileWriter(ALLOC_FILE));
- out.println("");
- out.println("");
- out.println(" ");
- out.println(" 1024mb,1vcores");
- out.println(" ");
- out.println("");
- out.close();
+ AllocationFileWriter.create()
+ .addQueue(new AllocationFileQueue.Builder("queueA")
+ .parent(true)
+ .maxChildResources("1024mb,1vcores")
+ .build())
+ .writeToFile(ALLOC_FILE);
scheduler.reinitialize(conf, resourceManager.getRMContext());
@@ -564,17 +558,13 @@ public void testChildMaxResources() throws IOException {
public void testFairShareWithZeroWeight() throws IOException {
conf.set(FairSchedulerConfiguration.ALLOCATION_FILE, ALLOC_FILE);
// set queueA and queueB weight zero.
- PrintWriter out = new PrintWriter(new FileWriter(ALLOC_FILE));
- out.println("");
- out.println("");
- out.println("");
- out.println("0.0");
- out.println("");
- out.println("");
- out.println("0.0");
- out.println("");
- out.println("");
- out.close();
+
+ AllocationFileWriter.create()
+ .addQueue(new AllocationFileQueue.Builder("queueA")
+ .weight(0.0f).build())
+ .addQueue(new AllocationFileQueue.Builder("queueB")
+ .weight(0.0f).build())
+ .writeToFile(ALLOC_FILE);
scheduler.init(conf);
scheduler.start();
@@ -612,26 +602,23 @@ public void testFairShareWithZeroWeight() throws IOException {
@Test
public void testComputeMaxAMResource() throws IOException {
conf.set(FairSchedulerConfiguration.ALLOCATION_FILE, ALLOC_FILE);
- PrintWriter out = new PrintWriter(new FileWriter(ALLOC_FILE));
- out.println("");
- out.println("");
- out.println("");
- out.println("0");
- out.println("0.5");
- out.println("4096 mb 4 vcores");
- out.println("");
- out.println("");
- out.println("0.0");
- out.println("0.5");
- out.println("");
- out.println("");
- out.println("1");
- out.println("0.5");
- out.println("");
- out.println("drf" +
- "");
- out.println("");
- out.close();
+
+ AllocationFileWriter.create()
+ .addQueue(new AllocationFileQueue.Builder("queueFSZeroWithMax")
+ .weight(0)
+ .maxAMShare(0.5)
+ .maxResources("4096 mb 4 vcores")
+ .build())
+ .addQueue(new AllocationFileQueue.Builder("queueFSZeroWithAVL")
+ .weight(0.0f)
+ .maxAMShare(0.5)
+ .build())
+ .addQueue(new AllocationFileQueue.Builder("queueFSNonZero")
+ .weight(1)
+ .maxAMShare(0.5)
+ .build())
+ .defaultQueueSchedulingPolicy("drf")
+ .writeToFile(ALLOC_FILE);
scheduler.init(conf);
scheduler.start();
@@ -746,19 +733,16 @@ public void testFairShareWithZeroWeightNoneZeroMinRes() throws IOException {
conf.set(FairSchedulerConfiguration.ALLOCATION_FILE, ALLOC_FILE);
// set queueA and queueB weight zero.
// set queueA and queueB minResources 1.
- PrintWriter out = new PrintWriter(new FileWriter(ALLOC_FILE));
- out.println("");
- out.println("");
- out.println("");
- out.println("1 mb 1 vcores");
- out.println("0.0");
- out.println("");
- out.println("");
- out.println("1 mb 1 vcores");
- out.println("0.0");
- out.println("");
- out.println("");
- out.close();
+ AllocationFileWriter.create()
+ .addQueue(new AllocationFileQueue.Builder("queueA")
+ .weight(0)
+ .minResources("1 mb 1 vcores")
+ .build())
+ .addQueue(new AllocationFileQueue.Builder("queueB")
+ .minResources("1 mb 1 vcores")
+ .weight(0.0f)
+ .build())
+ .writeToFile(ALLOC_FILE);
scheduler.init(conf);
scheduler.start();
@@ -796,19 +780,16 @@ public void testFairShareWithNoneZeroWeightNoneZeroMinRes()
conf.set(FairSchedulerConfiguration.ALLOCATION_FILE, ALLOC_FILE);
// set queueA and queueB weight 0.5.
// set queueA and queueB minResources 1024.
- PrintWriter out = new PrintWriter(new FileWriter(ALLOC_FILE));
- out.println("");
- out.println("");
- out.println("");
- out.println("1024 mb 1 vcores");
- out.println("0.5");
- out.println("");
- out.println("");
- out.println("1024 mb 1 vcores");
- out.println("0.5");
- out.println("");
- out.println("");
- out.close();
+ AllocationFileWriter.create()
+ .addQueue(new AllocationFileQueue.Builder("queueA")
+ .weight(0.5f)
+ .minResources("1024 mb 1 vcores")
+ .build())
+ .addQueue(new AllocationFileQueue.Builder("queueB")
+ .weight(0.5f)
+ .minResources("1024 mb 1 vcores")
+ .build())
+ .writeToFile(ALLOC_FILE);
scheduler.init(conf);
scheduler.start();
@@ -844,17 +825,14 @@ public void testFairShareWithNoneZeroWeightNoneZeroMinRes()
public void testQueueInfo() throws IOException {
conf.set(FairSchedulerConfiguration.ALLOCATION_FILE, ALLOC_FILE);
- PrintWriter out = new PrintWriter(new FileWriter(ALLOC_FILE));
- out.println("");
- out.println("");
- out.println("");
- out.println(".25");
- out.println("");
- out.println("");
- out.println(".75");
- out.println("");
- out.println("");
- out.close();
+ AllocationFileWriter.create()
+ .addQueue(new AllocationFileQueue.Builder("queueA")
+ .weight(0.25f)
+ .build())
+ .addQueue(new AllocationFileQueue.Builder("queueB")
+ .weight(0.75f)
+ .build())
+ .writeToFile(ALLOC_FILE);
scheduler.init(conf);
scheduler.start();
@@ -1367,19 +1345,15 @@ public void testReservationThresholdWithAssignMultiple() throws Exception {
public void testContainerReservationAttemptExceedingQueueMax()
throws Exception {
conf.set(FairSchedulerConfiguration.ALLOCATION_FILE, ALLOC_FILE);
- PrintWriter out = new PrintWriter(new FileWriter(ALLOC_FILE));
- out.println("");
- out.println("");
- out.println("");
- out.println("");
- out.println("2048mb,5vcores");
- out.println("");
- out.println("");
- out.println("2048mb,10vcores");
- out.println("");
- out.println("");
- out.println("");
- out.close();
+
+ AllocationFileWriter.create()
+ .addQueue(new AllocationFileQueue.Builder("root")
+ .subQueue(new AllocationFileQueue.Builder("queue1")
+ .maxResources("2048mb,5vcores").build())
+ .subQueue(new AllocationFileQueue.Builder("queue2")
+ .maxResources("2048mb,10vcores").build())
+ .build())
+ .writeToFile(ALLOC_FILE);
scheduler.init(conf);
scheduler.start();
@@ -1429,22 +1403,20 @@ public void testContainerReservationAttemptExceedingQueueMax()
public void testRequestAMResourceInZeroFairShareQueue() throws Exception {
conf.set(FairSchedulerConfiguration.ALLOCATION_FILE, ALLOC_FILE);
- PrintWriter out = new PrintWriter(new FileWriter(ALLOC_FILE));
- out.println("");
- out.println("");
- out.println("");
- out.println("0.0");
- out.println("4096mb,10vcores");
- out.println("0.5");
- out.println("");
- out.println("");
- out.println("2.0");
- out.println("");
- out.println("");
- out.println("0.000001");
- out.println("");
- out.println("");
- out.close();
+ AllocationFileWriter.create()
+ .addQueue(new AllocationFileQueue.Builder("queue1")
+ .weight(0)
+ .maxAMShare(0.5)
+ .maxResources("4096mb,10vcores")
+ .build())
+ .addQueue(new AllocationFileQueue.Builder("queue2")
+ .weight(2.0f)
+ .build())
+ .addQueue(new AllocationFileQueue.Builder("queue3")
+ .weight(0.000001f)
+ .build())
+ .defaultQueueSchedulingPolicy("drf")
+ .writeToFile(ALLOC_FILE);
scheduler.init(conf);
scheduler.start();
@@ -1502,19 +1474,15 @@ public void testRequestAMResourceInZeroFairShareQueue() throws Exception {
@Test (timeout = 500000)
public void testContainerReservationNotExceedingQueueMax() throws Exception {
conf.set(FairSchedulerConfiguration.ALLOCATION_FILE, ALLOC_FILE);
- PrintWriter out = new PrintWriter(new FileWriter(ALLOC_FILE));
- out.println("");
- out.println("");
- out.println("");
- out.println("");
- out.println("3072mb,10vcores");
- out.println("");
- out.println("");
- out.println("2048mb,10vcores");
- out.println("");
- out.println("");
- out.println("");
- out.close();
+
+ AllocationFileWriter.create()
+ .addQueue(new AllocationFileQueue.Builder("root")
+ .subQueue(new AllocationFileQueue.Builder("queue1")
+ .maxResources("3072mb,10vcores").build())
+ .subQueue(new AllocationFileQueue.Builder("queue2")
+ .maxResources("2048mb,10vcores").build())
+ .build())
+ .writeToFile(ALLOC_FILE);
scheduler.init(conf);
scheduler.start();
@@ -1563,19 +1531,14 @@ public void testContainerReservationNotExceedingQueueMax() throws Exception {
getCurrentReservation().getMemorySize());
// Now reduce max Resources of queue1 down to 2048
- out = new PrintWriter(new FileWriter(ALLOC_FILE));
- out.println("");
- out.println("");
- out.println("");
- out.println("");
- out.println("2048mb,10vcores");
- out.println("");
- out.println("");
- out.println("2048mb,10vcores");
- out.println("");
- out.println("");
- out.println("");
- out.close();
+ AllocationFileWriter.create()
+ .addQueue(new AllocationFileQueue.Builder("root")
+ .subQueue(new AllocationFileQueue.Builder("queue1")
+ .maxResources("2048mb,10vcores").build())
+ .subQueue(new AllocationFileQueue.Builder("queue2")
+ .maxResources("2048mb,10vcores").build())
+ .build())
+ .writeToFile(ALLOC_FILE);
scheduler.reinitialize(conf, resourceManager.getRMContext());
@@ -1596,13 +1559,9 @@ public void testContainerReservationNotExceedingQueueMax() throws Exception {
public void testReservationThresholdGatesReservations() throws Exception {
conf.set(FairSchedulerConfiguration.ALLOCATION_FILE, ALLOC_FILE);
- PrintWriter out = new PrintWriter(new FileWriter(ALLOC_FILE));
- out.println("");
- out.println("");
- out.println("drf" +
- "");
- out.println("");
- out.close();
+ AllocationFileWriter.create()
+ .defaultQueueSchedulingPolicy("drf")
+ .writeToFile(ALLOC_FILE);
// Set threshold to 2 * 1024 ==> 2048 MB & 2 * 1 ==> 2 vcores (test will
// use vcores)
@@ -1857,17 +1816,13 @@ public void testQueuePlacementWithPolicy() throws Exception {
public void testFairShareWithMinAlloc() throws Exception {
conf.set(FairSchedulerConfiguration.ALLOCATION_FILE, ALLOC_FILE);
- PrintWriter out = new PrintWriter(new FileWriter(ALLOC_FILE));
- out.println("");
- out.println("");
- out.println("");
- out.println("1024mb,0vcores");
- out.println("");
- out.println("");
- out.println("2048mb,0vcores");
- out.println("");
- out.println("");
- out.close();
+ AllocationFileWriter.create()
+ .addQueue(new AllocationFileQueue.Builder("queueA")
+ .minResources("1024mb,0vcores").build())
+ .addQueue(new AllocationFileQueue.Builder("queueB")
+ .minResources("2048mb,0vcores")
+ .build())
+ .writeToFile(ALLOC_FILE);
scheduler.init(conf);
scheduler.start();
@@ -1934,21 +1889,18 @@ public void testNestedUserQueue() throws IOException {
public void testFairShareAndWeightsInNestedUserQueueRule() throws Exception {
conf.set(FairSchedulerConfiguration.ALLOCATION_FILE, ALLOC_FILE);
- PrintWriter out = new PrintWriter(new FileWriter(ALLOC_FILE));
-
- out.println("");
- out.println("");
- out.println("");
- out.println("1024mb,0vcores");
- out.println("");
- out.println("");
- out.println("");
- out.println(" ");
- out.println("");
- out.println("");
- out.println("");
- out.println("");
- out.close();
+ AllocationFileWriter.create()
+ .addQueue(new AllocationFileQueue.Builder("parentq")
+ .parent(true)
+ .minResources("1024mb,0vcores")
+ .build())
+ .queuePlacementPolicy(new AllocationFileQueuePlacementPolicy()
+ .addRule(new AllocationFileQueuePlacementRule("nestedUserQueue")
+ .addNestedRule(
+ new AllocationFileQueuePlacementRule("specified")
+ .create(false)))
+ .addRule(new AllocationFileQueuePlacementRule("default")))
+ .writeToFile(ALLOC_FILE);
RMApp rmApp1 = new MockRMApp(0, 0, RMAppState.NEW);
RMApp rmApp2 = new MockRMApp(1, 1, RMAppState.NEW);
@@ -1995,21 +1947,16 @@ public void testFairShareAndWeightsInNestedUserQueueRule() throws Exception {
public void testSteadyFairShareWithReloadAndNodeAddRemove() throws Exception {
conf.set(FairSchedulerConfiguration.ALLOCATION_FILE, ALLOC_FILE);
- PrintWriter out = new PrintWriter(new FileWriter(ALLOC_FILE));
- out.println("");
- out.println("");
- out.println("fair");
- out.println("");
- out.println(" drf");
- out.println(" ");
- out.println(" 1");
- out.println(" ");
- out.println(" ");
- out.println(" 1");
- out.println(" ");
- out.println("");
- out.println("");
- out.close();
+ AllocationFileWriter.create()
+ .defaultQueueSchedulingPolicy("fair")
+ .addQueue(new AllocationFileQueue.Builder("root")
+ .schedulingPolicy("drf")
+ .subQueue(new AllocationFileQueue.Builder("child1")
+ .weight(1).build())
+ .subQueue(new AllocationFileQueue.Builder("child2")
+ .weight(1).build())
+ .build())
+ .writeToFile(ALLOC_FILE);
scheduler.init(conf);
scheduler.start();
@@ -2037,24 +1984,19 @@ public void testSteadyFairShareWithReloadAndNodeAddRemove() throws Exception {
.getSteadyFairShare().getMemorySize());
// Reload the allocation configuration file
- out = new PrintWriter(new FileWriter(ALLOC_FILE));
- out.println("");
- out.println("");
- out.println("fair");
- out.println("");
- out.println(" drf");
- out.println(" ");
- out.println(" 1");
- out.println(" ");
- out.println(" ");
- out.println(" 2");
- out.println(" ");
- out.println(" ");
- out.println(" 2");
- out.println(" ");
- out.println("");
- out.println("");
- out.close();
+ AllocationFileWriter.create()
+ .defaultQueueSchedulingPolicy("fair")
+ .addQueue(new AllocationFileQueue.Builder("root")
+ .schedulingPolicy("drf")
+ .subQueue(new AllocationFileQueue.Builder("child1")
+ .weight(1).build())
+ .subQueue(new AllocationFileQueue.Builder("child2")
+ .weight(2).build())
+ .subQueue(new AllocationFileQueue.Builder("child3")
+ .weight(2).build())
+ .build())
+ .writeToFile(ALLOC_FILE);
+
scheduler.reinitialize(conf, resourceManager.getRMContext());
// The steady fair shares for all queues should be updated
@@ -2171,27 +2113,19 @@ public void testQueueDemandCalculation() throws Exception {
}
@Test
- public void testHierarchicalQueueAllocationFileParsing() throws IOException, SAXException,
- AllocationConfigurationException, ParserConfigurationException {
+ public void testHierarchicalQueueAllocationFileParsing() throws IOException {
conf.set(FairSchedulerConfiguration.ALLOCATION_FILE, ALLOC_FILE);
-
- PrintWriter out = new PrintWriter(new FileWriter(ALLOC_FILE));
- out.println("");
- out.println("");
- out.println("");
- out.println("2048mb,0vcores");
- out.println("");
- out.println("");
- out.println("2048mb,0vcores");
- out.println("");
- out.println("2048mb,0vcores");
- out.println("");
- out.println("");
- out.println("2048mb,0vcores");
- out.println("");
- out.println("");
- out.println("");
- out.close();
+ AllocationFileWriter.create()
+ .addQueue(new AllocationFileQueue.Builder("queueA")
+ .minResources("2048mb,0vcores").build())
+ .addQueue(new AllocationFileQueue.Builder("queueB")
+ .minResources("2048mb,0vcores")
+ .subQueue(new AllocationFileQueue.Builder("queueC")
+ .minResources("2048mb,0vcores").build())
+ .subQueue(new AllocationFileQueue.Builder("queueD")
+ .minResources("2048mb,0vcores").build())
+ .build())
+ .writeToFile(ALLOC_FILE);
scheduler.init(conf);
scheduler.start();
@@ -2212,27 +2146,22 @@ public void testHierarchicalQueueAllocationFileParsing() throws IOException, SAX
public void testConfigureRootQueue() throws Exception {
conf.set(FairSchedulerConfiguration.ALLOCATION_FILE, ALLOC_FILE);
- PrintWriter out = new PrintWriter(new FileWriter(ALLOC_FILE));
- out.println("");
- out.println("");
- out.println("fair");
- out.println("");
- out.println(" drf");
- out.println(" ");
- out.println(" 1024mb,1vcores");
- out.println(" ");
- out.println(" ");
- out.println(" 1024mb,4vcores");
- out.println(" ");
- out.println(" 100");
- out.println(" 120");
- out.println(" .5");
- out.println("");
- out.println("300");
- out.println("200");
- out.println(".6");
- out.println("");
- out.close();
+ AllocationFileWriter.create()
+ .defaultQueueSchedulingPolicy("fair")
+ .defaultFairSharePreemptionTimeout(300)
+ .defaultMinSharePreemptionTimeout(200)
+ .defaultFairSharePreemptionThreshold(.6)
+ .addQueue(new AllocationFileQueue.Builder("root")
+ .schedulingPolicy("drf")
+ .fairSharePreemptionTimeout(100)
+ .fairSharePreemptionThreshold(.5)
+ .minSharePreemptionTimeout(120)
+ .subQueue(new AllocationFileQueue.Builder("child1")
+ .minResources("1024mb,1vcores").build())
+ .subQueue(new AllocationFileQueue.Builder("child2")
+ .minResources("1024mb,4vcores").build())
+ .build())
+ .writeToFile(ALLOC_FILE);
scheduler.init(conf);
scheduler.start();
@@ -2287,14 +2216,10 @@ public void testUserMaxRunningApps() throws Exception {
// Set max running apps
conf.set(FairSchedulerConfiguration.ALLOCATION_FILE, ALLOC_FILE);
- PrintWriter out = new PrintWriter(new FileWriter(ALLOC_FILE));
- out.println("");
- out.println("");
- out.println("");
- out.println("1");
- out.println("");
- out.println("");
- out.close();
+ AllocationFileWriter.create()
+ .userSettings(new UserSettings.Builder("user1")
+ .maxRunningApps(1).build())
+ .writeToFile(ALLOC_FILE);
scheduler.init(conf);
scheduler.start();
@@ -2339,64 +2264,56 @@ public void testUserMaxRunningApps() throws Exception {
@Test (timeout = 5000)
public void testIncreaseQueueMaxRunningAppsOnTheFly() throws Exception {
- String allocBefore = "" +
- "" +
- "" +
- "" +
- "1" +
- "" +
- "" +
- "";
-
- String allocAfter = "" +
- "" +
- "" +
- "" +
- "3" +
- "" +
- "" +
- "";
+ AllocationFileWriter allocBefore = AllocationFileWriter.create()
+ .addQueue(new AllocationFileQueue.Builder("root")
+ .subQueue(
+ new AllocationFileQueue.Builder("queue1")
+ .maxRunningApps(1)
+ .build())
+ .build());
+
+ AllocationFileWriter allocAfter = AllocationFileWriter.create()
+ .addQueue(new AllocationFileQueue.Builder("root")
+ .subQueue(
+ new AllocationFileQueue.Builder("queue1")
+ .maxRunningApps(3)
+ .build())
+ .build());
testIncreaseQueueSettingOnTheFlyInternal(allocBefore, allocAfter);
}
@Test (timeout = 5000)
public void testIncreaseUserMaxRunningAppsOnTheFly() throws Exception {
- String allocBefore = ""+
- ""+
- ""+
- ""+
- "10"+
- ""+
- ""+
- ""+
- "1"+
- ""+
- "";
-
- String allocAfter = ""+
- ""+
- ""+
- ""+
- "10"+
- ""+
- ""+
- ""+
- "3"+
- ""+
- "";
+ AllocationFileWriter allocBefore = AllocationFileWriter.create()
+ .addQueue(new AllocationFileQueue.Builder("root")
+ .subQueue(
+ new AllocationFileQueue.Builder("queue1")
+ .maxRunningApps(10)
+ .build())
+ .build())
+ .userSettings(new UserSettings.Builder("user1")
+ .maxRunningApps(1).build());
+
+ AllocationFileWriter allocAfter = AllocationFileWriter.create()
+ .addQueue(new AllocationFileQueue.Builder("root")
+ .subQueue(
+ new AllocationFileQueue.Builder("queue1")
+ .maxRunningApps(10)
+ .build())
+ .build())
+ .userSettings(new UserSettings.Builder("user1")
+ .maxRunningApps(3).build());
testIncreaseQueueSettingOnTheFlyInternal(allocBefore, allocAfter);
}
- private void testIncreaseQueueSettingOnTheFlyInternal(String allocBefore,
- String allocAfter) throws Exception {
+ private void testIncreaseQueueSettingOnTheFlyInternal(
+ AllocationFileWriter allocBefore,
+ AllocationFileWriter allocAfter) throws Exception {
// Set max running apps
conf.set(FairSchedulerConfiguration.ALLOCATION_FILE, ALLOC_FILE);
-
- PrintWriter out = new PrintWriter(new FileWriter(ALLOC_FILE));
- out.println(allocBefore);
- out.close();
+ allocBefore.writeToFile(ALLOC_FILE);
scheduler.init(conf);
scheduler.start();
@@ -2445,9 +2362,7 @@ private void testIncreaseQueueSettingOnTheFlyInternal(String allocBefore,
// App 4 should not be running
assertEquals(0, scheduler.getSchedulerApp(attId4).getLiveContainers().size());
- out = new PrintWriter(new FileWriter(ALLOC_FILE));
- out.println(allocAfter);
- out.close();
+ allocAfter.writeToFile(ALLOC_FILE);
scheduler.reinitialize(conf, resourceManager.getRMContext());
scheduler.update();
@@ -2482,65 +2397,57 @@ private void testIncreaseQueueSettingOnTheFlyInternal(String allocBefore,
@Test (timeout = 5000)
public void testDecreaseQueueMaxRunningAppsOnTheFly() throws Exception {
- String allocBefore = "" +
- "" +
- "" +
- "" +
- "3" +
- "" +
- "" +
- "";
-
- String allocAfter = "" +
- "" +
- "" +
- "" +
- "1" +
- "" +
- "" +
- "";
+ AllocationFileWriter allocBefore = AllocationFileWriter.create()
+ .addQueue(new AllocationFileQueue.Builder("root")
+ .subQueue(
+ new AllocationFileQueue.Builder("queue1")
+ .maxRunningApps(3)
+ .build())
+ .build());
+
+ AllocationFileWriter allocAfter = AllocationFileWriter.create()
+ .addQueue(new AllocationFileQueue.Builder("root")
+ .subQueue(
+ new AllocationFileQueue.Builder("queue1")
+ .maxRunningApps(1)
+ .build())
+ .build());
testDecreaseQueueSettingOnTheFlyInternal(allocBefore, allocAfter);
}
@Test (timeout = 5000)
public void testDecreaseUserMaxRunningAppsOnTheFly() throws Exception {
- String allocBefore = ""+
- ""+
- ""+
- ""+
- "10"+
- ""+
- ""+
- ""+
- "3"+
- ""+
- "";
-
- String allocAfter = ""+
- ""+
- ""+
- ""+
- "10"+
- ""+
- ""+
- ""+
- "1"+
- ""+
- "";
+ AllocationFileWriter allocBefore = AllocationFileWriter.create()
+ .addQueue(new AllocationFileQueue.Builder("root")
+ .subQueue(
+ new AllocationFileQueue.Builder("queue1")
+ .maxRunningApps(10)
+ .build())
+ .build())
+ .userSettings(new UserSettings.Builder("user1")
+ .maxRunningApps(3).build());
+
+ AllocationFileWriter allocAfter = AllocationFileWriter.create()
+ .addQueue(new AllocationFileQueue.Builder("root")
+ .subQueue(
+ new AllocationFileQueue.Builder("queue1")
+ .maxRunningApps(10)
+ .build())
+ .build())
+ .userSettings(new UserSettings.Builder("user1")
+ .maxRunningApps(1).build());
testDecreaseQueueSettingOnTheFlyInternal(allocBefore, allocAfter);
}
- private void testDecreaseQueueSettingOnTheFlyInternal(String allocBefore,
- String allocAfter) throws Exception {
+ private void testDecreaseQueueSettingOnTheFlyInternal(
+ AllocationFileWriter allocBefore,
+ AllocationFileWriter allocAfter) throws Exception {
// Set max running apps
conf.set(FairSchedulerConfiguration.ALLOCATION_FILE, ALLOC_FILE);
- PrintWriter out = new PrintWriter(new FileWriter(ALLOC_FILE));
- out.println(allocBefore);
- out.close();
-
+ allocBefore.writeToFile(ALLOC_FILE);
scheduler.init(conf);
scheduler.start();
scheduler.reinitialize(conf, resourceManager.getRMContext());
@@ -2588,9 +2495,7 @@ private void testDecreaseQueueSettingOnTheFlyInternal(String allocBefore,
// App 4 should not be running
assertEquals(0, scheduler.getSchedulerApp(attId4).getLiveContainers().size());
- out = new PrintWriter(new FileWriter(ALLOC_FILE));
- out.println(allocAfter);
- out.close();
+ allocAfter.writeToFile(ALLOC_FILE);
scheduler.reinitialize(conf, resourceManager.getRMContext());
scheduler.update();
@@ -2703,19 +2608,16 @@ public void testAclSubmitApplication() throws Exception {
// Set acl's
conf.set(FairSchedulerConfiguration.ALLOCATION_FILE, ALLOC_FILE);
- PrintWriter out = new PrintWriter(new FileWriter(ALLOC_FILE));
- out.println("");
- out.println("");
- out.println("");
- out.println(" ");
- out.println(" ");
- out.println(" ");
- out.println(" norealuserhasthisname");
- out.println(" norealuserhasthisname");
- out.println(" ");
- out.println("");
- out.println("");
- out.close();
+ AllocationFileWriter.create()
+ .addQueue(new AllocationFileQueue.Builder("root")
+ .aclSubmitApps(" ")
+ .aclAdministerApps(" ")
+ .subQueue(new AllocationFileQueue.Builder("queue1")
+ .aclSubmitApps("norealuserhasthisname")
+ .aclAdministerApps("norealuserhasthisname")
+ .build())
+ .build())
+ .writeToFile(ALLOC_FILE);
scheduler.init(conf);
scheduler.start();
@@ -3040,19 +2942,17 @@ public void testAssignContainer() throws Exception {
public void testNotAllowSubmitApplication() throws Exception {
// Set acl's
conf.set(FairSchedulerConfiguration.ALLOCATION_FILE, ALLOC_FILE);
- PrintWriter out = new PrintWriter(new FileWriter(ALLOC_FILE));
- out.println("");
- out.println("");
- out.println("");
- out.println(" ");
- out.println(" ");
- out.println(" ");
- out.println(" userallow");
- out.println(" userallow");
- out.println(" ");
- out.println("");
- out.println("");
- out.close();
+
+ AllocationFileWriter.create()
+ .addQueue(new AllocationFileQueue.Builder("root")
+ .aclSubmitApps(" ")
+ .aclAdministerApps(" ")
+ .subQueue(new AllocationFileQueue.Builder("queue1")
+ .aclSubmitApps("userallow")
+ .aclAdministerApps("userallow")
+ .build())
+ .build())
+ .writeToFile(ALLOC_FILE);
scheduler.init(conf);
scheduler.start();
@@ -3587,17 +3487,12 @@ private void verifyQueueNumRunnable(String queueName, int numRunnableInQueue,
public void testUserAndQueueMaxRunningApps() throws Exception {
conf.set(FairSchedulerConfiguration.ALLOCATION_FILE, ALLOC_FILE);
- PrintWriter out = new PrintWriter(new FileWriter(ALLOC_FILE));
- out.println("");
- out.println("");
- out.println("");
- out.println("2");
- out.println("");
- out.println("");
- out.println("1");
- out.println("");
- out.println("");
- out.close();
+ AllocationFileWriter.create()
+ .addQueue(new AllocationFileQueue.Builder("queue1")
+ .maxRunningApps(2).build())
+ .userSettings(new UserSettings.Builder("user1")
+ .maxRunningApps(1).build())
+ .writeToFile(ALLOC_FILE);
scheduler.init(conf);
scheduler.start();
@@ -3640,14 +3535,10 @@ public void testMultipleCompletedEvent() throws Exception {
// Set up a fair scheduler
conf.set(FairSchedulerConfiguration.ALLOCATION_FILE, ALLOC_FILE);
- PrintWriter out = new PrintWriter(new FileWriter(ALLOC_FILE));
- out.println("");
- out.println("");
- out.println("");
- out.println("0.2");
- out.println("");
- out.println("");
- out.close();
+ AllocationFileWriter.create()
+ .addQueue(new AllocationFileQueue.Builder("queue1")
+ .maxAMShare(0.2).build())
+ .writeToFile(ALLOC_FILE);
scheduler.init(conf);
scheduler.start();
@@ -3690,14 +3581,10 @@ public void testMultipleCompletedEvent() throws Exception {
public void testQueueMaxAMShare() throws Exception {
conf.set(FairSchedulerConfiguration.ALLOCATION_FILE, ALLOC_FILE);
- PrintWriter out = new PrintWriter(new FileWriter(ALLOC_FILE));
- out.println("");
- out.println("");
- out.println("");
- out.println("0.2");
- out.println("");
- out.println("");
- out.close();
+ AllocationFileWriter.create()
+ .addQueue(new AllocationFileQueue.Builder("queue1")
+ .maxAMShare(0.2).build())
+ .writeToFile(ALLOC_FILE);
scheduler.init(conf);
scheduler.start();
@@ -3920,25 +3807,18 @@ public void testQueueMaxAMShareDefault() throws Exception {
conf.set(FairSchedulerConfiguration.ALLOCATION_FILE, ALLOC_FILE);
conf.setInt(YarnConfiguration.RM_SCHEDULER_MAXIMUM_ALLOCATION_VCORES, 6);
- PrintWriter out = new PrintWriter(new FileWriter(ALLOC_FILE));
- out.println("");
- out.println("");
- out.println("");
- out.println("");
- out.println("");
- out.println("0.4");
- out.println("");
- out.println("");
- out.println("10240 mb 4 vcores");
- out.println("");
- out.println("");
- out.println("");
- out.println("");
- out.println("");
- out.println(
- "fair");
- out.println("");
- out.close();
+ AllocationFileWriter.create()
+ .defaultQueueSchedulingPolicy("fair")
+ .addQueue(new AllocationFileQueue.Builder("queue1").build())
+ .addQueue(new AllocationFileQueue.Builder("queue2")
+ .maxAMShare(0.4f)
+ .build())
+ .addQueue(new AllocationFileQueue.Builder("queue3")
+ .maxResources("10240 mb 4 vcores")
+ .build())
+ .addQueue(new AllocationFileQueue.Builder("queue4").build())
+ .addQueue(new AllocationFileQueue.Builder("queue5").build())
+ .writeToFile(ALLOC_FILE);
scheduler.init(conf);
scheduler.start();
@@ -4081,14 +3961,10 @@ public void testQueueMaxAMShareDefault() throws Exception {
public void testQueueMaxAMShareWithContainerReservation() throws Exception {
conf.set(FairSchedulerConfiguration.ALLOCATION_FILE, ALLOC_FILE);
conf.setFloat(FairSchedulerConfiguration.RESERVABLE_NODES, 1f);
- PrintWriter out = new PrintWriter(new FileWriter(ALLOC_FILE));
- out.println("");
- out.println("");
- out.println("");
- out.println("0.5");
- out.println("");
- out.println("");
- out.close();
+ AllocationFileWriter.create()
+ .addQueue(new AllocationFileQueue.Builder("queue1")
+ .maxAMShare(0.5).build())
+ .writeToFile(ALLOC_FILE);
scheduler.init(conf);
scheduler.start();
@@ -4342,19 +4218,16 @@ public void testMaxRunningAppsHierarchicalQueues() throws Exception {
ControlledClock clock = new ControlledClock();
scheduler.setClock(clock);
- PrintWriter out = new PrintWriter(new FileWriter(ALLOC_FILE));
- out.println("");
- out.println("");
- out.println("");
- out.println(" 3");
- out.println(" ");
- out.println(" ");
- out.println(" ");
- out.println(" 1");
- out.println(" ");
- out.println("");
- out.println("");
- out.close();
+ AllocationFileWriter.create()
+ .addQueue(new AllocationFileQueue.Builder("queue1")
+ .maxRunningApps(3)
+ .subQueue(new AllocationFileQueue.Builder("sub1").build())
+ .subQueue(new AllocationFileQueue.Builder("sub2").build())
+ .subQueue(new AllocationFileQueue.Builder("sub3")
+ .maxRunningApps(1)
+ .build())
+ .build())
+ .writeToFile(ALLOC_FILE);
scheduler.init(conf);
scheduler.start();
@@ -4466,11 +4339,8 @@ public void testDefaultRuleInitializesProperlyWhenPolicyNotConfigured()
conf.setBoolean(FairSchedulerConfiguration.ALLOW_UNDECLARED_POOLS, false);
// Create an alloc file with no queue placement policy
- PrintWriter out = new PrintWriter(new FileWriter(ALLOC_FILE));
- out.println("");
- out.println("");
- out.println("");
- out.close();
+ AllocationFileWriter.create()
+ .writeToFile(ALLOC_FILE);
scheduler.init(conf);
scheduler.start();
@@ -4913,16 +4783,13 @@ public void testQueueNameWithTrailingSpace() throws Exception {
}
@Test
- public void testEmptyQueueNameInConfigFile() throws IOException {
+ public void testEmptyQueueNameInConfigFile() {
conf.set(FairSchedulerConfiguration.ALLOCATION_FILE, ALLOC_FILE);
// set empty queue name
- PrintWriter out = new PrintWriter(new FileWriter(ALLOC_FILE));
- out.println("");
- out.println("");
- out.println("");
- out.println("");
- out.println("");
- out.close();
+ AllocationFileWriter.create()
+ .addQueue(new AllocationFileQueue.Builder("").build())
+ .writeToFile(ALLOC_FILE);
+
try {
scheduler.init(conf);
Assert.fail("scheduler init should fail because" +
@@ -5160,6 +5027,8 @@ public void testRefreshQueuesWhenRMHA() throws Exception {
new HAServiceProtocol.StateChangeRequestInfo(
HAServiceProtocol.RequestSource.REQUEST_BY_USER);
+ //ensure ALLOC_FILE contains an 'empty' config
+ AllocationFileWriter.create().writeToFile(ALLOC_FILE);
// 1. start a standby RM, file 'ALLOC_FILE' is empty, so there is no queues
MockRM rm1 = new MockRM(conf, null);
rm1.init(conf);
@@ -5167,14 +5036,10 @@ public void testRefreshQueuesWhenRMHA() throws Exception {
rm1.getAdminService().transitionToStandby(requestInfo);
// 2. add a new queue "test_queue"
- PrintWriter out = new PrintWriter(new FileWriter(ALLOC_FILE));
- out.println("");
- out.println("");
- out.println("");
- out.println(" 3");
- out.println("");
- out.println("");
- out.close();
+ AllocationFileWriter.create()
+ .addQueue(new AllocationFileQueue.Builder("test_queue")
+ .maxRunningApps(3).build())
+ .writeToFile(ALLOC_FILE);
conf.set(YarnConfiguration.RM_STORE, MemoryRMStateStore.class.getName());
// 3. start a active RM
@@ -5310,16 +5175,12 @@ public void testUpdateDemand() throws IOException {
public void testDumpState() throws IOException {
conf.set(FairSchedulerConfiguration.ALLOCATION_FILE, ALLOC_FILE);
- PrintWriter out = new PrintWriter(new FileWriter(ALLOC_FILE));
- out.println("");
- out.println("");
- out.println("");
- out.println(" ");
- out.println(" 1");
- out.println(" ");
- out.println("");
- out.println("");
- out.close();
+ AllocationFileWriter.create()
+ .addQueue(new AllocationFileQueue.Builder("parent")
+ .subQueue(new AllocationFileQueue.Builder("child1")
+ .weight(1).build())
+ .build())
+ .writeToFile(ALLOC_FILE);
ControlledClock clock = new ControlledClock();
scheduler.setClock(clock);
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFairSchedulerFairShare.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFairSchedulerFairShare.java
index a79aacc196e..c1925fda6f6 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFairSchedulerFairShare.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFairSchedulerFairShare.java
@@ -33,6 +33,11 @@
import org.apache.hadoop.yarn.server.resourcemanager.rmnode.RMNode;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.event.AppAttemptRemovedSchedulerEvent;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.event.NodeAddedSchedulerEvent;
+
+import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair
+ .allocationfile.AllocationFileQueue;
+import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair
+ .allocationfile.AllocationFileWriter;
import org.apache.hadoop.yarn.util.resource.Resources;
import org.junit.After;
import org.junit.Before;
@@ -57,34 +62,29 @@ public void teardown() {
conf = null;
}
- private void createClusterWithQueuesAndOneNode(int mem, String policy)
- throws IOException {
+ private void createClusterWithQueuesAndOneNode(int mem, String policy) {
createClusterWithQueuesAndOneNode(mem, 0, policy);
}
private void createClusterWithQueuesAndOneNode(int mem, int vCores,
- String policy) throws IOException {
- PrintWriter out = new PrintWriter(new FileWriter(ALLOC_FILE));
- out.println("");
- out.println("");
- out.println("");
- out.println(" ");
- out.println(" 8");
- out.println(" ");
- out.println(" ");
- out.println(" ");
- out.println(" ");
- out.println(" ");
- out.println(" ");
- out.println(" 1");
- out.println(" ");
- out.println(" ");
- out.println(" ");
- out.println("");
- out.println("" + policy
- + "");
- out.println("");
- out.close();
+ String policy) {
+ AllocationFileWriter.create()
+ .defaultQueueSchedulingPolicy(policy)
+ .addQueue(new AllocationFileQueue.Builder("root")
+ .subQueue(new AllocationFileQueue.Builder("parentA")
+ .weight(8)
+ .subQueue(new AllocationFileQueue.Builder("childA1").build())
+ .subQueue(new AllocationFileQueue.Builder("childA2").build())
+ .subQueue(new AllocationFileQueue.Builder("childA3").build())
+ .subQueue(new AllocationFileQueue.Builder("childA4").build())
+ .build())
+ .subQueue(new AllocationFileQueue.Builder("parentB")
+ .weight(1)
+ .subQueue(new AllocationFileQueue.Builder("childB1").build())
+ .subQueue(new AllocationFileQueue.Builder("childB2").build())
+ .build())
+ .build())
+ .writeToFile(ALLOC_FILE);
resourceManager = new MockRM(conf);
resourceManager.start();
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFairSchedulerQueueACLs.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFairSchedulerQueueACLs.java
index 32aab5531f8..041b09e3667 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFairSchedulerQueueACLs.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFairSchedulerQueueACLs.java
@@ -25,33 +25,34 @@
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.yarn.conf.YarnConfiguration;
import org.apache.hadoop.yarn.server.resourcemanager.QueueACLsTestBase;
+import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair
+ .allocationfile.AllocationFileQueue;
+import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair
+ .allocationfile.AllocationFileWriter;
public class TestFairSchedulerQueueACLs extends QueueACLsTestBase {
@Override
- protected Configuration createConfiguration() throws IOException {
+ protected Configuration createConfiguration() {
FairSchedulerConfiguration fsConf = new FairSchedulerConfiguration();
final String TEST_DIR = new File(System.getProperty("test.build.data",
"/tmp")).getAbsolutePath();
final String ALLOC_FILE = new File(TEST_DIR, "test-queues.xml")
.getAbsolutePath();
- PrintWriter out = new PrintWriter(new FileWriter(ALLOC_FILE));
- out.println("");
- out.println("");
- out.println("");
- out.println(" ");
- out.println(" root_admin ");
- out.println(" ");
- out.println(" queueA_user,common_user ");
- out.println(" queueA_admin ");
- out.println(" ");
- out.println(" ");
- out.println(" queueB_user,common_user ");
- out.println(" queueB_admin ");
- out.println(" ");
- out.println("");
- out.println("");
- out.close();
+
+ AllocationFileWriter.create()
+ .addQueue(new AllocationFileQueue.Builder("root")
+ .aclSubmitApps(" ")
+ .aclAdministerApps("root_admin ")
+ .subQueue(new AllocationFileQueue.Builder("queueA")
+ .aclSubmitApps("queueA_user,common_user ")
+ .aclAdministerApps("queueA_admin ").build())
+ .subQueue(new AllocationFileQueue.Builder("queueB")
+ .aclSubmitApps("queueB_user,common_user ")
+ .aclAdministerApps("queueB_admin ").build())
+ .build())
+ .writeToFile(ALLOC_FILE);
+
fsConf.set(FairSchedulerConfiguration.ALLOCATION_FILE, ALLOC_FILE);
fsConf.setBoolean(YarnConfiguration.YARN_ACL_ENABLE, true);
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestQueueManagerRealScheduler.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestQueueManagerRealScheduler.java
index 5736f75bbe5..4e4cc7a3d8c 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestQueueManagerRealScheduler.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestQueueManagerRealScheduler.java
@@ -18,6 +18,10 @@
package org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair;
import org.apache.hadoop.yarn.server.resourcemanager.MockRM;
+import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair
+ .allocationfile.AllocationFileQueue;
+import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair
+ .allocationfile.AllocationFileWriter;
import org.junit.After;
import org.junit.Before;
import org.junit.Test;
@@ -38,7 +42,7 @@
@Before
public void setup() throws IOException {
createConfiguration();
- writeAllocFile(30, 40);
+ writeAllocFile(30);
conf.set(FairSchedulerConfiguration.ALLOCATION_FILE,
ALLOC_FILE.getAbsolutePath());
@@ -56,32 +60,20 @@ public void teardown() {
}
}
- private void writeAllocFile(int defaultFairShareTimeout,
- int fairShareTimeout) throws IOException {
- PrintWriter out = new PrintWriter(new FileWriter(ALLOC_FILE));
- out.println("");
- out.println("");
- out.println("");
- out.println("");
- out.println("");
- out.println("");
- out.println("");
- out.println("");
- out.println("5");
- out.println("");
- out.println("");
- out.println("");
- out.println("");
- out.println("");
- out.println("");
- out.println("15"
- + "");
- out.println("" +
- + defaultFairShareTimeout + "");
- out.println(""
- + fairShareTimeout + "");
- out.println("");
- out.close();
+ private void writeAllocFile(int defaultFairShareTimeout) {
+ AllocationFileWriter.create()
+ .addQueue(new AllocationFileQueue.Builder("default")
+ .build())
+ .addQueue(new AllocationFileQueue.Builder("queueA").build())
+ .addQueue(new AllocationFileQueue.Builder("queueB")
+ .subQueue(new AllocationFileQueue.Builder("queueB1")
+ .minSharePreemptionTimeout(5).build())
+ .subQueue(new AllocationFileQueue.Builder("queueB2").build())
+ .build())
+ .addQueue(new AllocationFileQueue.Builder("queueC").build())
+ .defaultMinSharePreemptionTimeout(15)
+ .defaultFairSharePreemptionTimeout(defaultFairShareTimeout)
+ .writeToFile(ALLOC_FILE.getAbsolutePath());
}
@Test
@@ -120,7 +112,7 @@ public void testBackwardsCompatiblePreemptionConfiguration()
// Lower the fairshare preemption timeouts and verify it is picked
// correctly.
- writeAllocFile(25, 30);
+ writeAllocFile(25);
scheduler.reinitialize(conf, resourceManager.getRMContext());
assertEquals(25000, queueMgr.getQueue("root")
.getFairSharePreemptionTimeout());
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestSchedulingPolicy.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestSchedulingPolicy.java
index b016c1b4fb8..027d39d39e5 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestSchedulingPolicy.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestSchedulingPolicy.java
@@ -30,6 +30,8 @@
import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.yarn.api.records.Priority;
import org.apache.hadoop.yarn.api.records.Resource;
+import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.allocationfile.AllocationFileQueue;
+import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.allocationfile.AllocationFileWriter;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.policies.DominantResourceFairnessPolicy;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.policies.FairSharePolicy;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.policies.FifoPolicy;
@@ -309,22 +311,16 @@ public boolean isPreemptable() {
@Test
public void testSchedulingPolicyViolation() throws IOException {
conf.set(FairSchedulerConfiguration.ALLOCATION_FILE, ALLOC_FILE);
- PrintWriter out = new PrintWriter(new FileWriter(ALLOC_FILE));
- out.println("");
- out.println("");
- out.println("");
- out.println("fair");
- out.println(" ");
- out.println(" drf");
- out.println(" ");
- out.println(" ");
- out.println(" fair");
- out.println(" ");
- out.println("");
- out.println("drf" +
- "");
- out.println("");
- out.close();
+ AllocationFileWriter.create()
+ .defaultQueueSchedulingPolicy("drf")
+ .addQueue(new AllocationFileQueue.Builder("root")
+ .schedulingPolicy("fair")
+ .subQueue(new AllocationFileQueue.Builder("child1")
+ .schedulingPolicy("drf").build())
+ .subQueue(new AllocationFileQueue.Builder("child2")
+ .schedulingPolicy("fair").build())
+ .build())
+ .writeToFile(ALLOC_FILE);
scheduler.init(conf);
@@ -339,22 +335,16 @@ public void testSchedulingPolicyViolation() throws IOException {
+ " policy if its parent policy is 'fair'.", dynamicQueue);
// Set child1 to 'fair' and child2 to 'drf', the reload the allocation file.
- out = new PrintWriter(new FileWriter(ALLOC_FILE));
- out.println("");
- out.println("");
- out.println("");
- out.println("fair");
- out.println(" ");
- out.println(" fair");
- out.println(" ");
- out.println(" ");
- out.println(" drf");
- out.println(" ");
- out.println("");
- out.println("drf" +
- "");
- out.println("");
- out.close();
+ AllocationFileWriter.create()
+ .defaultQueueSchedulingPolicy("drf")
+ .addQueue(new AllocationFileQueue.Builder("root")
+ .schedulingPolicy("fair")
+ .subQueue(new AllocationFileQueue.Builder("child1")
+ .schedulingPolicy("fair").build())
+ .subQueue(new AllocationFileQueue.Builder("child2")
+ .schedulingPolicy("drf").build())
+ .build())
+ .writeToFile(ALLOC_FILE);
scheduler.reinitialize(conf, null);
child1 = scheduler.getQueueManager().getQueue("child1");
@@ -369,26 +359,21 @@ public void testSchedulingPolicyViolation() throws IOException {
}
@Test
- public void testSchedulingPolicyViolationInTheMiddleLevel()
- throws IOException {
+ public void testSchedulingPolicyViolationInTheMiddleLevel() {
conf.set(FairSchedulerConfiguration.ALLOCATION_FILE, ALLOC_FILE);
- PrintWriter out = new PrintWriter(new FileWriter(ALLOC_FILE));
- out.println("");
- out.println("");
- out.println("");
- out.println("fair");
- out.println(" ");
- out.println(" fair");
- out.println(" ");
- out.println(" drf");
- out.println(" ");
- out.println(" fair");
- out.println(" ");
- out.println(" ");
- out.println(" ");
- out.println("");
- out.println("");
- out.close();
+ AllocationFileWriter.create()
+ .addQueue(new AllocationFileQueue.Builder("root")
+ .schedulingPolicy("fair")
+ .subQueue(new AllocationFileQueue.Builder("level2")
+ .schedulingPolicy("fair")
+ .subQueue(new AllocationFileQueue.Builder("level3")
+ .schedulingPolicy("drf")
+ .subQueue(new AllocationFileQueue.Builder("leaf")
+ .schedulingPolicy("fair").build())
+ .build())
+ .build())
+ .build())
+ .writeToFile(ALLOC_FILE);
scheduler.init(conf);
@@ -407,19 +392,16 @@ public void testSchedulingPolicyViolationInTheMiddleLevel()
public void testFIFOPolicyOnlyForLeafQueues()
throws IOException {
conf.set(FairSchedulerConfiguration.ALLOCATION_FILE, ALLOC_FILE);
- PrintWriter out = new PrintWriter(new FileWriter(ALLOC_FILE));
- out.println("");
- out.println("");
- out.println("");
- out.println(" ");
- out.println(" fifo");
- out.println(" ");
- out.println(" fair");
- out.println(" ");
- out.println(" ");
- out.println("");
- out.println("");
- out.close();
+
+ AllocationFileWriter.create()
+ .addQueue(new AllocationFileQueue.Builder("root")
+ .subQueue(new AllocationFileQueue.Builder("intermediate")
+ .schedulingPolicy("fifo")
+ .subQueue(new AllocationFileQueue.Builder("leaf")
+ .schedulingPolicy("fair").build())
+ .build())
+ .build())
+ .writeToFile(ALLOC_FILE);
scheduler.init(conf);
@@ -427,19 +409,15 @@ public void testFIFOPolicyOnlyForLeafQueues()
assertNull("Queue 'intermediate' should be null since 'fifo' is only for "
+ "leaf queue.", intermediate);
- out = new PrintWriter(new FileWriter(ALLOC_FILE));
- out.println("");
- out.println("");
- out.println("");
- out.println(" ");
- out.println(" fair");
- out.println(" ");
- out.println(" fifo");
- out.println(" ");
- out.println(" ");
- out.println("");
- out.println("");
- out.close();
+ AllocationFileWriter.create()
+ .addQueue(new AllocationFileQueue.Builder("root")
+ .subQueue(new AllocationFileQueue.Builder("intermediate")
+ .schedulingPolicy("fair")
+ .subQueue(new AllocationFileQueue.Builder("leaf")
+ .schedulingPolicy("fifo").build())
+ .build())
+ .build())
+ .writeToFile(ALLOC_FILE);
scheduler.reinitialize(conf, null);
@@ -453,39 +431,28 @@ public void testFIFOPolicyOnlyForLeafQueues()
@Test
public void testPolicyReinitilization() throws IOException {
conf.set(FairSchedulerConfiguration.ALLOCATION_FILE, ALLOC_FILE);
- PrintWriter out = new PrintWriter(new FileWriter(ALLOC_FILE));
- out.println("");
- out.println("");
- out.println("");
- out.println("fair");
- out.println(" ");
- out.println(" fair");
- out.println(" ");
- out.println(" ");
- out.println(" fair");
- out.println(" ");
- out.println("");
- out.println("");
- out.close();
+ AllocationFileWriter.create()
+ .addQueue(new AllocationFileQueue.Builder("root")
+ .schedulingPolicy("fair")
+ .subQueue(new AllocationFileQueue.Builder("child1")
+ .schedulingPolicy("fair").build())
+ .subQueue(new AllocationFileQueue.Builder("child2")
+ .schedulingPolicy("fair").build())
+ .build())
+ .writeToFile(ALLOC_FILE);
scheduler.init(conf);
// Set child1 to 'drf' which is not allowed, then reload the allocation file
- out = new PrintWriter(new FileWriter(ALLOC_FILE));
- out.println("");
- out.println("");
- out.println("");
- out.println("fair");
- out.println(" ");
- out.println(" drf");
- out.println(" ");
- out.println(" ");
- out.println(" fifo");
- out.println(" ");
- out.println("");
- out.println("");
- out.close();
-
+ AllocationFileWriter.create()
+ .addQueue(new AllocationFileQueue.Builder("root")
+ .schedulingPolicy("fair")
+ .subQueue(new AllocationFileQueue.Builder("child1")
+ .schedulingPolicy("drf").build())
+ .subQueue(new AllocationFileQueue.Builder("child2")
+ .schedulingPolicy("fifo").build())
+ .build())
+ .writeToFile(ALLOC_FILE);
scheduler.reinitialize(conf, null);
FSQueue child1 = scheduler.getQueueManager().getQueue("child1");
@@ -498,20 +465,15 @@ public void testPolicyReinitilization() throws IOException {
child2.getPolicy() instanceof FairSharePolicy);
// Set both child1 and root to 'drf', then reload the allocation file
- out = new PrintWriter(new FileWriter(ALLOC_FILE));
- out.println("");
- out.println("");
- out.println("");
- out.println("drf");
- out.println(" ");
- out.println(" drf");
- out.println(" ");
- out.println(" ");
- out.println(" fifo");
- out.println(" ");
- out.println("");
- out.println("");
- out.close();
+ AllocationFileWriter.create()
+ .addQueue(new AllocationFileQueue.Builder("root")
+ .schedulingPolicy("drf")
+ .subQueue(new AllocationFileQueue.Builder("child1")
+ .schedulingPolicy("drf").build())
+ .subQueue(new AllocationFileQueue.Builder("child2")
+ .schedulingPolicy("fifo").build())
+ .build())
+ .writeToFile(ALLOC_FILE);
scheduler.reinitialize(conf, null);
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/allocationfile/AllocationFileQueue.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/allocationfile/AllocationFileQueue.java
index f1afe6979fc..0dfd6511f2f 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/allocationfile/AllocationFileQueue.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/allocationfile/AllocationFileQueue.java
@@ -16,67 +16,249 @@
package org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.allocationfile;
+import com.google.common.collect.Lists;
+
import java.io.PrintWriter;
import java.io.StringWriter;
import java.util.List;
-class AllocationFileQueue {
- private final AllocationFileQueueProperties properties;
+import static org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair
+ .allocationfile.AllocationFileWriter.addIfPresent;
+
+public class AllocationFileQueue {
+ private static final String DEFAULT_TAG_NAME = "queue";
+ private static final String LEGACY_TAG_NAME = "pool";
+
+ private final String queueName;
+ private final String minResources;
+ private final String maxResources;
+ private final String aclAdministerApps;
+ private final String aclSubmitApps;
+ private final String aclSubmitReservations;
+ private final String aclAdministerReservations;
+ private final String aclListReservations;
+ private final String schedulingPolicy;
+ private final Integer maxRunningApps;
+ private final Double maxAMShare;
+ private final Integer minSharePreemptionTimeout;
+ private final String maxChildResources;
+ private final Integer fairSharePreemptionTimeout;
+ private final Double fairSharePreemptionThreshold;
+ private final String maxContainerAllocation;
private final List subQueues;
+ private final Float weight;
+ private String tagName;
+
+ private final boolean parent;
+ private final boolean reservation;
- AllocationFileQueue(AllocationFileQueueProperties properties,
- List subQueues) {
- this.properties = properties;
- this.subQueues = subQueues;
+ private AllocationFileQueue(Builder builder) {
+ this.queueName = builder.name;
+ this.parent = builder.parent;
+ this.minResources = builder.minResources;
+ this.maxResources = builder.maxResources;
+ this.aclAdministerApps = builder.aclAdministerApps;
+ this.aclSubmitApps = builder.aclSubmitApps;
+ this.aclSubmitReservations = builder.aclSubmitReservations;
+ this.aclAdministerReservations = builder.aclAdministerReservations;
+ this.aclListReservations = builder.aclListReservations;
+ this.schedulingPolicy = builder.schedulingPolicy;
+ this.maxRunningApps = builder.maxRunningApps;
+ this.maxAMShare = builder.maxAMShare;
+ this.minSharePreemptionTimeout = builder.minSharePreemptionTimeout;
+ this.maxChildResources = builder.maxChildResources;
+ this.fairSharePreemptionTimeout = builder.fairSharePreemptionTimeout;
+ this.fairSharePreemptionThreshold = builder.fairSharePreemptionThreshold;
+ this.maxContainerAllocation = builder.maxContainerAllocation;
+ this.weight = builder.weight;
+ this.reservation = builder.reservation;
+ this.subQueues = builder.subQueues;
+ this.tagName = DEFAULT_TAG_NAME;
}
String render() {
StringWriter sw = new StringWriter();
PrintWriter pw = new PrintWriter(sw);
printStartTag(pw);
- AllocationFileWriter.printQueues(pw, subQueues);
- AllocationFileWriter.addIfPresent(pw, "minResources",
- properties::getMinResources);
- AllocationFileWriter.addIfPresent(pw, "maxResources",
- properties::getMaxResources);
- AllocationFileWriter.addIfPresent(pw, "aclAdministerApps",
- properties::getAclAdministerApps);
- AllocationFileWriter.addIfPresent(pw, "aclSubmitApps",
- properties::getAclSubmitApps);
- AllocationFileWriter.addIfPresent(pw, "schedulingPolicy",
- properties::getSchedulingPolicy);
- AllocationFileWriter.addIfPresent(pw, "maxRunningApps",
- () -> AllocationFileWriter
- .createNumberSupplier(properties.getMaxRunningApps()));
- AllocationFileWriter.addIfPresent(pw, "maxAMShare",
- () -> AllocationFileWriter.createNumberSupplier(properties
- .getMaxAMShare()));
- AllocationFileWriter.addIfPresent(pw, "minSharePreemptionTimeout",
- () -> AllocationFileWriter
- .createNumberSupplier(properties.getMinSharePreemptionTimeout()));
- AllocationFileWriter.addIfPresent(pw, "maxChildResources",
- properties::getMaxChildResources);
- AllocationFileWriter.addIfPresent(pw, "fairSharePreemptionTimeout",
- () -> AllocationFileWriter
- .createNumberSupplier(properties.getFairSharePreemptionTimeout()));
- AllocationFileWriter.addIfPresent(pw, "fairSharePreemptionThreshold",
- () -> AllocationFileWriter
- .createNumberSupplier(
- properties.getFairSharePreemptionThreshold()));
+ AllocationFileWriter.printQueues(pw, subQueues,
+ tagName.equals(LEGACY_TAG_NAME));
+ addIfPresent(pw, "minResources", minResources);
+ addIfPresent(pw, "maxResources", maxResources);
+ addIfPresent(pw, "aclAdministerApps", aclAdministerApps);
+ addIfPresent(pw, "aclSubmitApps", aclSubmitApps);
+ addIfPresent(pw, "aclSubmitReservations", aclSubmitReservations);
+ addIfPresent(pw, "aclAdministerReservations", aclAdministerReservations);
+ addIfPresent(pw, "aclListReservations", aclListReservations);
+ addIfPresent(pw, "schedulingPolicy", schedulingPolicy);
+ addIfPresent(pw, "maxRunningApps", maxRunningApps);
+ addIfPresent(pw, "maxAMShare", maxAMShare);
+ addIfPresent(pw, "minSharePreemptionTimeout", minSharePreemptionTimeout);
+ addIfPresent(pw, "maxChildResources", maxChildResources);
+ addIfPresent(pw, "fairSharePreemptionTimeout", fairSharePreemptionTimeout);
+ addIfPresent(pw, "fairSharePreemptionThreshold",
+ fairSharePreemptionThreshold);
+ addIfPresent(pw, "maxContainerAllocation", maxContainerAllocation);
+ addIfPresent(pw, "weight", weight);
+ if (reservation) {
+ pw.println("");
+ }
printEndTag(pw);
pw.close();
+
return sw.toString();
}
+ String renderWithLegacyTag() {
+ this.tagName = LEGACY_TAG_NAME;
+ return render();
+ }
+
private void printStartTag(PrintWriter pw) {
- pw.print("");
}
private void printEndTag(PrintWriter pw) {
- pw.println("");
+ pw.println("" + tagName + ">");
+ }
+
+ /**
+ * Class that can build queues (with subqueues) for testcases.
+ * The intention of having this class to group the common properties of
+ * simple queues and subqueues by methods delegating calls to a
+ * queuePropertiesBuilder instance.
+ */
+ public static class Builder {
+ String name;
+ String minResources;
+ String maxResources;
+ String aclAdministerApps;
+ String aclSubmitApps;
+ String aclSubmitReservations;
+ String aclAdministerReservations;
+ String aclListReservations;
+ String schedulingPolicy;
+ Integer maxRunningApps;
+ Double maxAMShare;
+ Integer minSharePreemptionTimeout;
+ boolean parent;
+ String maxChildResources;
+ Integer fairSharePreemptionTimeout;
+ Double fairSharePreemptionThreshold;
+ String maxContainerAllocation;
+ boolean reservation;
+ final List subQueues = Lists.newArrayList();
+ private Float weight;
+
+ public Builder(String name) {
+ this.name = name;
+ }
+
+ public Builder parent(boolean parent) {
+ this.parent = parent;
+ return this;
+ }
+
+ public Builder minResources(String value) {
+ this.minResources = value;
+ return this;
+ }
+
+ public Builder maxResources(String value) {
+ this.maxResources = value;
+ return this;
+ }
+
+ public Builder aclAdministerApps(String value) {
+ this.aclAdministerApps = value;
+ return this;
+ }
+
+ public Builder aclSubmitApps(String value) {
+ this.aclSubmitApps = value;
+ return this;
+ }
+
+ public Builder aclSubmitReservations(String value) {
+ this.aclSubmitReservations = value;
+ return this;
+ }
+
+ public Builder aclAdministerReservations(String value) {
+ this.aclAdministerReservations = value;
+ return this;
+ }
+
+ public Builder aclListReservations(String value) {
+ this.aclListReservations = value;
+ return this;
+ }
+
+ public Builder schedulingPolicy(String value) {
+ this.schedulingPolicy = value;
+ return this;
+ }
+
+ public Builder maxRunningApps(int value) {
+ this.maxRunningApps = value;
+ return this;
+ }
+
+ public Builder maxAMShare(double value) {
+ this.maxAMShare = value;
+ return this;
+ }
+
+ public Builder minSharePreemptionTimeout(int value) {
+ this.minSharePreemptionTimeout = value;
+ return this;
+ }
+
+ public Builder maxChildResources(String value) {
+ this.maxChildResources = value;
+ return this;
+ }
+
+ public Builder fairSharePreemptionTimeout(Integer value) {
+ this.fairSharePreemptionTimeout = value;
+ return this;
+ }
+
+ public Builder fairSharePreemptionThreshold(
+ double value) {
+ this.fairSharePreemptionThreshold = value;
+ return this;
+ }
+
+ public Builder maxContainerAllocation(String value) {
+ this.maxContainerAllocation = value;
+ return this;
+ }
+
+ public Builder weight(float value) {
+ this.weight = value;
+ return this;
+ }
+
+ public Builder reservation() {
+ this.reservation = true;
+ return this;
+ }
+
+ public Builder subQueue(AllocationFileQueue queue) {
+ if (queue == null) {
+ throw new IllegalArgumentException("Subqueue cannot be null!");
+ }
+ subQueues.add(queue);
+ return this;
+ }
+
+ public AllocationFileQueue build() {
+ return new AllocationFileQueue(this);
+ }
}
}
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/allocationfile/AllocationFileQueueBuilder.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/allocationfile/AllocationFileQueueBuilder.java
deleted file mode 100644
index a2faf1da318..00000000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/allocationfile/AllocationFileQueueBuilder.java
+++ /dev/null
@@ -1,115 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- * http://www.apache.org/licenses/LICENSE-2.0
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.allocationfile;
-
-/**
- * Abstract base class for building simple queues and subqueues for testcases.
- * Currently there are two concrete types subclassed from this class:
- * {@link AllocationFileSimpleQueueBuilder} and
- * {@link AllocationFileSubQueueBuilder}.
- * The intention of having this class to group the common properties of
- * simple queues and subqueues by methods delegating calls to a
- * queuePropertiesBuilder instance.
- */
-public abstract class AllocationFileQueueBuilder {
- final AllocationFileQueueProperties.Builder queuePropertiesBuilder;
-
- AllocationFileQueueBuilder() {
- this.queuePropertiesBuilder =
- AllocationFileQueueProperties.Builder.create();
- }
-
- public AllocationFileQueueBuilder parent(boolean parent) {
- this.queuePropertiesBuilder.parent(parent);
- return this;
- }
-
- public AllocationFileQueueBuilder minResources(String value) {
- this.queuePropertiesBuilder.minResources(value);
- return this;
- }
-
- public AllocationFileQueueBuilder maxResources(String value) {
- this.queuePropertiesBuilder.maxResources(value);
- return this;
- }
-
- public AllocationFileQueueBuilder aclAdministerApps(String value) {
- this.queuePropertiesBuilder.aclAdministerApps(value);
- return this;
- }
-
- public AllocationFileQueueBuilder aclSubmitApps(String value) {
- this.queuePropertiesBuilder.aclSubmitApps(value);
- return this;
- }
-
- public AllocationFileQueueBuilder schedulingPolicy(String value) {
- this.queuePropertiesBuilder.schedulingPolicy(value);
- return this;
- }
-
- public AllocationFileQueueBuilder maxRunningApps(int value) {
- this.queuePropertiesBuilder.maxRunningApps(value);
- return this;
- }
-
- public AllocationFileQueueBuilder maxAMShare(double value) {
- this.queuePropertiesBuilder.maxAMShare(value);
- return this;
- }
-
- public AllocationFileQueueBuilder minSharePreemptionTimeout(int value) {
- this.queuePropertiesBuilder.minSharePreemptionTimeout(value);
- return this;
- }
-
- public AllocationFileQueueBuilder maxChildResources(String value) {
- this.queuePropertiesBuilder.maxChildResources(value);
- return this;
- }
-
- public AllocationFileQueueBuilder fairSharePreemptionTimeout(Integer value) {
- this.queuePropertiesBuilder.fairSharePreemptionTimeout(value);
- return this;
- }
-
- public AllocationFileQueueBuilder fairSharePreemptionThreshold(
- double value) {
- this.queuePropertiesBuilder.fairSharePreemptionThreshold(value);
- return this;
- }
-
- public AllocationFileQueueBuilder subQueue(String queueName) {
- if (this instanceof AllocationFileSimpleQueueBuilder) {
- return new AllocationFileSubQueueBuilder(
- (AllocationFileSimpleQueueBuilder) this, queueName);
- } else {
- throw new IllegalStateException(
- "subQueue can only be invoked on instances of "
- + AllocationFileSimpleQueueBuilder.class);
- }
- }
-
- public abstract AllocationFileWriter buildQueue();
-
- public abstract AllocationFileSimpleQueueBuilder buildSubQueue();
-
- AllocationFileQueueProperties.Builder getqueuePropertiesBuilder() {
- return queuePropertiesBuilder;
- }
-}
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/allocationfile/AllocationFileQueuePlacementPolicy.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/allocationfile/AllocationFileQueuePlacementPolicy.java
new file mode 100644
index 00000000000..0d0b9565bff
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/allocationfile/AllocationFileQueuePlacementPolicy.java
@@ -0,0 +1,59 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.allocationfile;
+
+
+import com.google.common.collect.Lists;
+
+import java.io.PrintWriter;
+import java.io.StringWriter;
+import java.util.List;
+
+public class AllocationFileQueuePlacementPolicy {
+ private List rules = Lists.newArrayList();
+
+ public AllocationFileQueuePlacementPolicy addRule(
+ AllocationFileQueuePlacementRule rule) {
+ this.rules.add(rule);
+ return this;
+ }
+
+ public String render() {
+ StringWriter sw = new StringWriter();
+ PrintWriter pw = new PrintWriter(sw);
+ addStartTag(pw);
+ addRules(pw);
+ addEndTag(pw);
+ pw.close();
+
+ return sw.toString();
+ }
+
+ private void addStartTag(PrintWriter pw) {
+ pw.println("");
+ }
+
+ private void addRules(PrintWriter pw) {
+ for (AllocationFileQueuePlacementRule rule : rules) {
+ pw.println(rule.render());
+ }
+ }
+
+ private void addEndTag(PrintWriter pw) {
+ pw.println("");
+ }
+}
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/allocationfile/AllocationFileQueuePlacementRule.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/allocationfile/AllocationFileQueuePlacementRule.java
new file mode 100644
index 00000000000..cbc1d872534
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/allocationfile/AllocationFileQueuePlacementRule.java
@@ -0,0 +1,84 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.allocationfile;
+
+
+import com.google.common.collect.Lists;
+
+import java.io.PrintWriter;
+import java.io.StringWriter;
+import java.util.List;
+
+public class AllocationFileQueuePlacementRule {
+ private String name;
+ private boolean create = true;
+ private String queue;
+ private List nestedRules = Lists
+ .newArrayList();
+
+ public AllocationFileQueuePlacementRule(String name) {
+ this.name = name;
+ }
+
+ public AllocationFileQueuePlacementRule create(boolean create) {
+ this.create = create;
+ return this;
+ }
+
+ public AllocationFileQueuePlacementRule queue(String queue) {
+ this.queue = queue;
+ return this;
+ }
+
+ public AllocationFileQueuePlacementRule addNestedRule(
+ AllocationFileQueuePlacementRule rule) {
+ this.nestedRules.add(rule);
+ return this;
+ }
+
+ public String render() {
+ StringWriter sw = new StringWriter();
+ PrintWriter pw = new PrintWriter(sw);
+ addStartTag(pw);
+ addNestedRules(pw);
+ addEndTag(pw);
+ pw.close();
+
+ return sw.toString();
+ }
+
+ private void addStartTag(PrintWriter pw) {
+ pw.print("");
+ }
+
+ private void addNestedRules(PrintWriter pw) {
+ if (nestedRules != null && !nestedRules.isEmpty()) {
+ for (AllocationFileQueuePlacementRule rule : nestedRules) {
+ pw.println(rule.render());
+ }
+ }
+ }
+
+ private void addEndTag(PrintWriter pw) {
+ pw.println("");
+ }
+}
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/allocationfile/AllocationFileQueueProperties.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/allocationfile/AllocationFileQueueProperties.java
deleted file mode 100644
index 2c01144a152..00000000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/allocationfile/AllocationFileQueueProperties.java
+++ /dev/null
@@ -1,202 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- * http://www.apache.org/licenses/LICENSE-2.0
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.allocationfile;
-
-/**
- * The purpose of this class is to store all properties of a queue.
- */
-public class AllocationFileQueueProperties {
- private final String queueName;
- private final String minResources;
- private final String maxResources;
- private final String aclAdministerApps;
- private final String aclSubmitApps;
- private final String schedulingPolicy;
- private final Integer maxRunningApps;
- private final Double maxAMShare;
- private final Integer minSharePreemptionTimeout;
- private final Boolean parent;
- private final String maxChildResources;
- private final Integer fairSharePreemptionTimeout;
- private final Double fairSharePreemptionThreshold;
-
- AllocationFileQueueProperties(Builder builder) {
- this.queueName = builder.queueName;
- this.parent = builder.parent;
- this.minResources = builder.minResources;
- this.maxResources = builder.maxResources;
- this.aclAdministerApps = builder.aclAdministerApps;
- this.aclSubmitApps = builder.aclSubmitApps;
- this.schedulingPolicy = builder.schedulingPolicy;
- this.maxRunningApps = builder.maxRunningApps;
- this.maxAMShare = builder.maxAMShare;
- this.minSharePreemptionTimeout = builder.minSharePreemptionTimeout;
- this.maxChildResources = builder.maxChildResources;
- this.fairSharePreemptionTimeout = builder.fairSharePreemptionTimeout;
- this.fairSharePreemptionThreshold = builder.fairSharePreemptionThreshold;
- }
-
- public String getQueueName() {
- return queueName;
- }
-
- public String getMinResources() {
- return minResources;
- }
-
- public String getMaxResources() {
- return maxResources;
- }
-
- public String getAclAdministerApps() {
- return aclAdministerApps;
- }
-
- public String getAclSubmitApps() {
- return aclSubmitApps;
- }
-
- public String getSchedulingPolicy() {
- return schedulingPolicy;
- }
-
- public Integer getMaxRunningApps() {
- return maxRunningApps;
- }
-
- public Double getMaxAMShare() {
- return maxAMShare;
- }
-
- public Integer getMinSharePreemptionTimeout() {
- return minSharePreemptionTimeout;
- }
-
- public Boolean getParent() {
- return parent;
- }
-
- public String getMaxChildResources() {
- return maxChildResources;
- }
-
- public Integer getFairSharePreemptionTimeout() {
- return fairSharePreemptionTimeout;
- }
-
- public Double getFairSharePreemptionThreshold() {
- return fairSharePreemptionThreshold;
- }
-
- /**
- * Builder class for {@link AllocationFileQueueProperties}.
- */
- public static final class Builder {
- private String queueName;
- private Boolean parent = false;
- private String minResources;
- private String maxResources;
- private String aclAdministerApps;
- private String aclSubmitApps;
- private String schedulingPolicy;
- private Integer maxRunningApps;
- private Double maxAMShare;
- private Integer minSharePreemptionTimeout;
- private String maxChildResources;
- private Integer fairSharePreemptionTimeout;
- private Double fairSharePreemptionThreshold;
-
- Builder() {
- }
-
- public static Builder create() {
- return new Builder();
- }
-
- public Builder queueName(String queueName) {
- this.queueName = queueName;
- return this;
- }
-
- public Builder minResources(String minResources) {
- this.minResources = minResources;
- return this;
- }
-
- public Builder maxResources(String maxResources) {
- this.maxResources = maxResources;
- return this;
- }
-
- public Builder aclAdministerApps(String aclAdministerApps) {
- this.aclAdministerApps = aclAdministerApps;
- return this;
- }
-
- public Builder aclSubmitApps(String aclSubmitApps) {
- this.aclSubmitApps = aclSubmitApps;
- return this;
- }
-
- public Builder schedulingPolicy(String schedulingPolicy) {
- this.schedulingPolicy = schedulingPolicy;
- return this;
- }
-
- public Builder maxRunningApps(Integer maxRunningApps) {
- this.maxRunningApps = maxRunningApps;
- return this;
- }
-
- public Builder maxAMShare(Double maxAMShare) {
- this.maxAMShare = maxAMShare;
- return this;
- }
-
- public Builder minSharePreemptionTimeout(
- Integer minSharePreemptionTimeout) {
- this.minSharePreemptionTimeout = minSharePreemptionTimeout;
- return this;
- }
-
- public Builder parent(Boolean parent) {
- this.parent = parent;
- return this;
- }
-
- public Builder maxChildResources(String maxChildResources) {
- this.maxChildResources = maxChildResources;
- return this;
- }
-
- public Builder fairSharePreemptionTimeout(
- Integer fairSharePreemptionTimeout) {
- this.fairSharePreemptionTimeout = fairSharePreemptionTimeout;
- return this;
- }
-
- public Builder fairSharePreemptionThreshold(
- Double fairSharePreemptionThreshold) {
- this.fairSharePreemptionThreshold = fairSharePreemptionThreshold;
- return this;
- }
-
- public AllocationFileQueueProperties build() {
- return new AllocationFileQueueProperties(this);
- }
- }
-}
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/allocationfile/AllocationFileSimpleQueueBuilder.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/allocationfile/AllocationFileSimpleQueueBuilder.java
deleted file mode 100644
index 93d100ed35c..00000000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/allocationfile/AllocationFileSimpleQueueBuilder.java
+++ /dev/null
@@ -1,64 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- * http://www.apache.org/licenses/LICENSE-2.0
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.allocationfile;
-
-import java.util.ArrayList;
-import java.util.List;
-
-/**
- * Queue builder that can build a simple queue with its properties.
- * Subqueues can be added with {@link #addSubQueue(AllocationFileQueue)}.
- */
-public class AllocationFileSimpleQueueBuilder
- extends AllocationFileQueueBuilder {
- private final AllocationFileWriter allocationFileWriter;
- private final List subQueues = new ArrayList<>();
-
- AllocationFileSimpleQueueBuilder(AllocationFileWriter allocationFileWriter,
- String queueName) {
- this.allocationFileWriter = allocationFileWriter;
- getqueuePropertiesBuilder().queueName(queueName);
- }
-
- void addSubQueue(AllocationFileQueue queue) {
- subQueues.add(queue);
- }
-
- @Override
- public AllocationFileWriter buildQueue() {
- AllocationFileQueueProperties queueProperties =
- getqueuePropertiesBuilder().build();
- AllocationFileQueue queue =
- new AllocationFileQueue(queueProperties, subQueues);
-
- if (allocationFileWriter != null) {
- allocationFileWriter.addQueue(queue);
- } else {
- throw new IllegalStateException(
- "allocationFileWriter field has to be set on a " + getClass());
- }
-
- return allocationFileWriter;
- }
-
- @Override
- public AllocationFileSimpleQueueBuilder buildSubQueue() {
- throw new IllegalStateException(
- "buildSubQueue is not supported in " + getClass());
- }
-
-}
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/allocationfile/AllocationFileSubQueueBuilder.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/allocationfile/AllocationFileSubQueueBuilder.java
deleted file mode 100644
index 728aedcbc82..00000000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/allocationfile/AllocationFileSubQueueBuilder.java
+++ /dev/null
@@ -1,54 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- * http://www.apache.org/licenses/LICENSE-2.0
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.allocationfile;
-
-import com.google.common.collect.Lists;
-
-
-/**
- * Queue builder that can build a subqueue with its properties.
- */
-public class AllocationFileSubQueueBuilder extends AllocationFileQueueBuilder {
- private AllocationFileSimpleQueueBuilder parentQueueBuilder;
-
- AllocationFileSubQueueBuilder(
- AllocationFileSimpleQueueBuilder parentQueueBuilder, String queueName) {
- getqueuePropertiesBuilder().queueName(queueName);
- this.parentQueueBuilder = parentQueueBuilder;
- }
-
- @Override
- public AllocationFileWriter buildQueue() {
- throw new IllegalStateException(
- "BuildQueue is not supported in " + getClass());
- }
-
- public AllocationFileSimpleQueueBuilder buildSubQueue() {
- AllocationFileQueueProperties queueProperties =
- getqueuePropertiesBuilder().build();
- AllocationFileQueue queue =
- new AllocationFileQueue(queueProperties, Lists.newArrayList());
-
- if (parentQueueBuilder != null) {
- parentQueueBuilder.addSubQueue(queue);
- return parentQueueBuilder;
- } else {
- throw new IllegalStateException(
- "parentQueueBuilder field has to be set on a " + getClass());
- }
- }
-}
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/allocationfile/AllocationFileWriter.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/allocationfile/AllocationFileWriter.java
index df1cc53d840..f41ca4eeac5 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/allocationfile/AllocationFileWriter.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/allocationfile/AllocationFileWriter.java
@@ -21,7 +21,6 @@
import java.io.PrintWriter;
import java.util.ArrayList;
import java.util.List;
-import java.util.function.Supplier;
/**
* This class is capable of serializing allocation file data to a file
@@ -39,6 +38,10 @@
private String defaultQueueSchedulingPolicy;
private List queues = new ArrayList<>();
private UserSettings userSettings;
+ private boolean useLegacyTagNameForQueues = false;
+ private String reservationAgent;
+ private String reservationPolicy;
+ private AllocationFileQueuePlacementPolicy queuePlacementPolicy;
private AllocationFileWriter() {
}
@@ -47,8 +50,9 @@ public static AllocationFileWriter create() {
return new AllocationFileWriter();
}
- public AllocationFileSimpleQueueBuilder queue(String queueName) {
- return new AllocationFileSimpleQueueBuilder(this, queueName);
+ public AllocationFileWriter addQueue(AllocationFileQueue queue) {
+ queues.add(queue);
+ return this;
}
public AllocationFileWriter queueMaxAppsDefault(int value) {
@@ -92,21 +96,41 @@ public AllocationFileWriter defaultQueueSchedulingPolicy(String value) {
return this;
}
- public UserSettings.Builder userSettings(String username) {
- return new UserSettings.Builder(this, username);
+ public AllocationFileWriter useLegacyTagNameForQueues() {
+ this.useLegacyTagNameForQueues = true;
+ return this;
+ }
+
+ public AllocationFileWriter reservationAgent(String value) {
+ this.reservationAgent = value;
+ return this;
+ }
+
+ public AllocationFileWriter reservationPolicy(String value) {
+ this.reservationPolicy = value;
+ return this;
}
- void addQueue(AllocationFileQueue queue) {
- this.queues.add(queue);
+ public AllocationFileWriter userSettings(UserSettings settings) {
+ this.userSettings = settings;
+ return this;
}
- void setUserSettings(UserSettings userSettings) {
- this.userSettings = userSettings;
+ public AllocationFileWriter queuePlacementPolicy(AllocationFileQueuePlacementPolicy policy) {
+ this.queuePlacementPolicy = policy;
+ return this;
}
- static void printQueues(PrintWriter pw, List queues) {
+ static void printQueues(PrintWriter pw, List queues,
+ boolean useLegacyTagName) {
for (AllocationFileQueue queue : queues) {
- pw.println(queue.render());
+ final String queueStr;
+ if (useLegacyTagName) {
+ queueStr = queue.renderWithLegacyTag();
+ } else {
+ queueStr = queue.render();
+ }
+ pw.println(queueStr);
}
}
@@ -114,22 +138,18 @@ private void printUserSettings(PrintWriter pw) {
pw.println(userSettings.render());
}
- static void addIfPresent(PrintWriter pw, String tag,
- Supplier supplier) {
- if (supplier.get() != null) {
- pw.println("<" + tag + ">" + supplier.get() + "" + tag + ">");
- }
+ private void printQueuePlacementPolicy(PrintWriter pw) {
+ pw.println(queuePlacementPolicy.render());
}
- static String createNumberSupplier(Object number) {
- if (number != null) {
- return number.toString();
+ static void addIfPresent(PrintWriter pw, String tag, Object obj) {
+ if (obj != null) {
+ pw.println("<" + tag + ">" + obj.toString() + "" + tag + ">");
}
- return null;
}
private void writeHeader(PrintWriter pw) {
- pw.println("");
+ pw.println("");
pw.println("");
}
@@ -140,34 +160,37 @@ private void writeFooter(PrintWriter pw) {
public void writeToFile(String filename) {
PrintWriter pw;
try {
- pw = new PrintWriter(new FileWriter(filename));
+ pw = new PrintWriter(filename, "UTF-8");
} catch (IOException e) {
throw new RuntimeException(e);
}
writeHeader(pw);
if (!queues.isEmpty()) {
- printQueues(pw, queues);
+ printQueues(pw, queues, useLegacyTagNameForQueues);
}
if (userSettings != null) {
printUserSettings(pw);
}
- addIfPresent(pw, "queueMaxAppsDefault",
- () -> createNumberSupplier(queueMaxAppsDefault));
- addIfPresent(pw, "queueMaxResourcesDefault",
- () -> queueMaxResourcesDefault);
- addIfPresent(pw, "userMaxAppsDefault",
- () -> createNumberSupplier(userMaxAppsDefault));
- addIfPresent(pw, "queueMaxAMShareDefault",
- () -> createNumberSupplier(queueMaxAMShareDefault));
+ if (queuePlacementPolicy != null) {
+ printQueuePlacementPolicy(pw);
+ }
+
+ addIfPresent(pw, "queueMaxAppsDefault", queueMaxAppsDefault);
+ addIfPresent(pw, "queueMaxResourcesDefault", queueMaxResourcesDefault);
+ addIfPresent(pw, "userMaxAppsDefault", userMaxAppsDefault);
+ addIfPresent(pw, "queueMaxAMShareDefault", queueMaxAMShareDefault);
addIfPresent(pw, "defaultMinSharePreemptionTimeout",
- () -> createNumberSupplier(defaultMinSharePreemptionTimeout));
+ defaultMinSharePreemptionTimeout);
addIfPresent(pw, "defaultFairSharePreemptionTimeout",
- () -> createNumberSupplier(defaultFairSharePreemptionTimeout));
+ defaultFairSharePreemptionTimeout);
addIfPresent(pw, "defaultFairSharePreemptionThreshold",
- () -> createNumberSupplier(defaultFairSharePreemptionThreshold));
+ defaultFairSharePreemptionThreshold);
addIfPresent(pw, "defaultQueueSchedulingPolicy",
- () -> defaultQueueSchedulingPolicy);
+ defaultQueueSchedulingPolicy);
+ addIfPresent(pw, "reservation-agent", reservationAgent);
+ addIfPresent(pw, "reservation-policy", reservationPolicy);
+
writeFooter(pw);
pw.close();
}
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/allocationfile/UserSettings.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/allocationfile/UserSettings.java
index 7a5656e5f3a..e8457d0e8ec 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/allocationfile/UserSettings.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/allocationfile/UserSettings.java
@@ -23,7 +23,7 @@
* Value class that stores user settings and can render data in XML format,
* see {@link #render()}.
*/
-class UserSettings {
+public class UserSettings {
private final String username;
private final Integer maxRunningApps;
@@ -36,8 +36,7 @@ public String render() {
StringWriter sw = new StringWriter();
PrintWriter pw = new PrintWriter(sw);
addStartTag(pw);
- AllocationFileWriter.addIfPresent(pw, "maxRunningApps",
- () -> AllocationFileWriter.createNumberSupplier(maxRunningApps));
+ AllocationFileWriter.addIfPresent(pw, "maxRunningApps", maxRunningApps);
addEndTag(pw);
pw.close();
@@ -56,12 +55,10 @@ private void addEndTag(PrintWriter pw) {
* Builder class for {@link UserSettings}
*/
public static class Builder {
- private final AllocationFileWriter allocationFileWriter;
private final String username;
private Integer maxRunningApps;
- Builder(AllocationFileWriter allocationFileWriter, String username) {
- this.allocationFileWriter = allocationFileWriter;
+ public Builder(String username) {
this.username = username;
}
@@ -70,11 +67,8 @@ public Builder maxRunningApps(int value) {
return this;
}
- public AllocationFileWriter build() {
- UserSettings userSettings = new UserSettings(this);
- allocationFileWriter.setUserSettings(userSettings);
-
- return allocationFileWriter;
+ public UserSettings build() {
+ return new UserSettings(this);
}
}
}
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesAppsModification.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesAppsModification.java
index 5d99e0d169d..d370a5951a5 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesAppsModification.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesAppsModification.java
@@ -81,6 +81,11 @@
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacitySchedulerConfiguration;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.FairScheduler;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.FairSchedulerConfiguration;
+
+import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair
+ .allocationfile.AllocationFileQueue;
+import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair
+ .allocationfile.AllocationFileWriter;
import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.AppPriority;
import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.AppQueue;
import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.AppState;
@@ -202,23 +207,15 @@ public void configureScheduler() {
private class FairTestServletModule extends TestServletModule {
@Override
public void configureScheduler() {
- try {
- PrintWriter out = new PrintWriter(new FileWriter(FS_ALLOC_FILE));
- out.println("");
- out.println("");
- out.println("");
- out.println(" someuser ");
- out.println(" ");
- out.println(" someuser ");
- out.println(" ");
- out.println(" ");
- out.println(" someuser ");
- out.println(" ");
- out.println("");
- out.println("");
- out.close();
- } catch(IOException e) {
- }
+ AllocationFileWriter.create()
+ .addQueue(new AllocationFileQueue.Builder("root")
+ .aclAdministerApps("someuser ")
+ .subQueue(new AllocationFileQueue.Builder("default")
+ .aclAdministerApps("someuser ").build())
+ .subQueue(new AllocationFileQueue.Builder("test")
+ .aclAdministerApps("someuser ").build())
+ .build())
+ .writeToFile(FS_ALLOC_FILE);
conf.set(FairSchedulerConfiguration.ALLOCATION_FILE, FS_ALLOC_FILE);
conf.set(YarnConfiguration.RM_SCHEDULER, FairScheduler.class.getName());
}
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesReservation.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesReservation.java
index 7e8ba88c88c..9ba643f8cb5 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesReservation.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesReservation.java
@@ -52,6 +52,11 @@
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacitySchedulerConfiguration;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.FairScheduler;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.FairSchedulerConfiguration;
+
+import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair
+ .allocationfile.AllocationFileQueue;
+import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair
+ .allocationfile.AllocationFileWriter;
import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.ReservationDeleteRequestInfo;
import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.ReservationSubmissionRequestInfo;
import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.ReservationUpdateRequestInfo;
@@ -180,27 +185,17 @@ public void configureScheduler() {
private static class FairTestServletModule extends TestServletModule {
@Override
public void configureScheduler() {
- try {
- PrintWriter out = new PrintWriter(new FileWriter(FS_ALLOC_FILE));
- out.println("");
- out.println("");
- out.println("");
- out.println(" someuser ");
- out.println(" ");
- out.println(" someuser ");
- out.println(" ");
- out.println(" ");
- out.println(" ");
- out.println(" ");
- out.println(" someuser ");
- out.println(" ");
- out.println("");
- out.println("drf" +
- "");
- out.println("");
- out.close();
- } catch (IOException e) {
- }
+ AllocationFileWriter.create()
+ .defaultQueueSchedulingPolicy("drf")
+ .addQueue(new AllocationFileQueue.Builder("root")
+ .aclAdministerApps("someuser ")
+ .subQueue(new AllocationFileQueue.Builder("default")
+ .aclAdministerApps("someuser ").build())
+ .subQueue(new AllocationFileQueue.Builder("dedicated")
+ .reservation()
+ .aclAdministerApps("someuser ").build())
+ .build())
+ .writeToFile(FS_ALLOC_FILE);
conf.set(FairSchedulerConfiguration.ALLOCATION_FILE, FS_ALLOC_FILE);
conf.set(YarnConfiguration.RM_SCHEDULER, FairScheduler.class.getName());
}