diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/AllocationFileLoaderService.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/AllocationFileLoaderService.java index 2022510..e0e23e0 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/AllocationFileLoaderService.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/AllocationFileLoaderService.java @@ -396,6 +396,12 @@ private void loadQueue(String parentName, Element element, Map> configuredQueues) throws AllocationConfigurationException { String queueName = element.getAttribute("name"); + + if (queueName.contains(".")) { + throw new AllocationConfigurationException("Bad fair scheduler config " + + "file: queue name (" + queueName + ") shouldn't contain period."); + } + if (parentName != null) { queueName = parentName + "." + queueName; } diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FairScheduler.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FairScheduler.java index 94fb849..1e728a4 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FairScheduler.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FairScheduler.java @@ -585,6 +585,17 @@ protected synchronized void addApplication(ApplicationId applicationId, return; } + if (queueName.startsWith(".") || queueName.endsWith(".")) { + String message = "Reject application " + applicationId + + " submitted by user " + user + " with an illegal queue name (" + + queueName + "). " + + "The queue name cannot start/end with period."; + LOG.info(message); + rmContext.getDispatcher().getEventHandler() + .handle(new RMAppRejectedEvent(applicationId, message)); + return; + } + RMApp rmApp = rmContext.getRMApps().get(applicationId); FSLeafQueue queue = assignToQueue(rmApp, queueName, user); if (queue == null) { diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/QueuePlacementRule.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/QueuePlacementRule.java index 056df57..409b8bc 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/QueuePlacementRule.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/QueuePlacementRule.java @@ -23,6 +23,8 @@ import java.util.Map; import java.util.Set; +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; import org.apache.hadoop.classification.InterfaceAudience.Private; import org.apache.hadoop.classification.InterfaceStability.Unstable; import org.apache.hadoop.security.Groups; @@ -38,7 +40,9 @@ @Unstable public abstract class QueuePlacementRule { protected boolean create; - + public static final Log LOG = + LogFactory.getLog(QueuePlacementRule.class.getName()); + /** * Initializes the rule with any arguments. * @@ -125,7 +129,7 @@ protected abstract String getQueueForApp(String requestedQueue, String user, @Override protected String getQueueForApp(String requestedQueue, String user, Groups groups, Map> configuredQueues) { - return "root." + user; + return "root." + convertUsername(user); } @Override @@ -241,7 +245,7 @@ protected String getQueueForApp(String requestedQueue, String user, if (configuredQueues.get(FSQueueType.LEAF).contains(queueName)) { return ""; } - return queueName + "." + user; + return queueName + "." + convertUsername(user); } return queueName; } @@ -339,4 +343,18 @@ public boolean isTerminal() { return true; } } + + /** + * Replace the periods in the username with "_dot_". + */ + protected String convertUsername(String user) { + if (user.contains(".")) { + String convertedUser = user.replaceAll("\\.", "_dot_"); + LOG.warn("Username " + user + " is converted to " + convertedUser + + " when it is used as a queue name."); + return convertedUser; + } else { + return user; + } + } } diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestAllocationFileLoaderService.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestAllocationFileLoaderService.java index 656e20d..9a66a94 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestAllocationFileLoaderService.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestAllocationFileLoaderService.java @@ -525,6 +525,30 @@ public void testQueueAlongsideRoot() throws Exception { allocLoader.setReloadListener(confHolder); allocLoader.reloadAllocations(); } + + /** + * Verify that you can't include periods as the queue name in the allocations + * file. + */ + @Test (expected = AllocationConfigurationException.class) + public void testQueueNameContainingPeriods() throws Exception { + Configuration conf = new Configuration(); + conf.set(FairSchedulerConfiguration.ALLOCATION_FILE, ALLOC_FILE); + + PrintWriter out = new PrintWriter(new FileWriter(ALLOC_FILE)); + out.println(""); + out.println(""); + out.println(""); + out.println(""); + out.println(""); + out.close(); + + AllocationFileLoaderService allocLoader = new AllocationFileLoaderService(); + allocLoader.init(conf); + ReloadListener confHolder = new ReloadListener(); + allocLoader.setReloadListener(confHolder); + allocLoader.reloadAllocations(); + } private class ReloadListener implements AllocationFileLoaderService.Listener { public AllocationConfiguration allocConf; diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFairScheduler.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFairScheduler.java index b5169bf..9506063 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFairScheduler.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFairScheduler.java @@ -916,6 +916,46 @@ public void testEmptyQueueName() throws Exception { } @Test + public void testQueueuNameWithPeriods() throws Exception { + scheduler.init(conf); + scheduler.start(); + scheduler.reinitialize(conf, resourceManager.getRMContext()); + + // only default queue + assertEquals(1, scheduler.getQueueManager().getLeafQueues().size()); + + // submit app with queue name (.A) + ApplicationAttemptId appAttemptId1 = createAppAttemptId(1, 1); + AppAddedSchedulerEvent appAddedEvent1 = + new AppAddedSchedulerEvent(appAttemptId1.getApplicationId(), ".A", "user1"); + scheduler.handle(appAddedEvent1); + // submission rejected + assertEquals(1, scheduler.getQueueManager().getLeafQueues().size()); + assertNull(scheduler.getSchedulerApp(appAttemptId1)); + assertEquals(0, resourceManager.getRMContext().getRMApps().size()); + + // submit app with queue name (A.) + ApplicationAttemptId appAttemptId2 = createAppAttemptId(2, 1); + AppAddedSchedulerEvent appAddedEvent2 = + new AppAddedSchedulerEvent(appAttemptId2.getApplicationId(), "A.", "user1"); + scheduler.handle(appAddedEvent2); + // submission rejected + assertEquals(1, scheduler.getQueueManager().getLeafQueues().size()); + assertNull(scheduler.getSchedulerApp(appAttemptId2)); + assertEquals(0, resourceManager.getRMContext().getRMApps().size()); + + // submit app with queue name (A.B) + ApplicationAttemptId appAttemptId3 = createAppAttemptId(3, 1); + AppAddedSchedulerEvent appAddedEvent3 = + new AppAddedSchedulerEvent(appAttemptId3.getApplicationId(), "A.B", "user1"); + scheduler.handle(appAddedEvent3); + // submission accepted + assertEquals(2, scheduler.getQueueManager().getLeafQueues().size()); + assertNull(scheduler.getSchedulerApp(appAttemptId3)); + assertEquals(0, resourceManager.getRMContext().getRMApps().size()); + } + + @Test public void testAssignToQueue() throws Exception { conf.set(FairSchedulerConfiguration.USER_AS_DEFAULT_QUEUE, "true"); scheduler.init(conf); diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestQueuePlacementPolicy.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestQueuePlacementPolicy.java index e20b0c3..bb81f86 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestQueuePlacementPolicy.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestQueuePlacementPolicy.java @@ -345,7 +345,31 @@ public void testNestedUserQueueDefaultRule() throws Exception { assertEquals("root.parentq.user1", policy.assignAppToQueue("root.default", "user1")); } - + + @Test + public void testUserContainsPeriod() throws Exception { + // This test covers the user case where the username contains periods. + StringBuffer sb = new StringBuffer(); + sb.append(""); + sb.append(" "); + sb.append(""); + QueuePlacementPolicy policy = parse(sb.toString()); + assertEquals("root.first_dot_last", + policy.assignAppToQueue("default", "first.last")); + + sb = new StringBuffer(); + sb.append(""); + sb.append(" "); + sb.append(" "); + sb.append(" "); + sb.append(" "); + sb.append(" "); + sb.append(""); + policy = parse(sb.toString()); + assertEquals("root.default.first_dot_last", + policy.assignAppToQueue("root.default", "first.last")); + } + private QueuePlacementPolicy parse(String str) throws Exception { // Read and parse the allocations file. DocumentBuilderFactory docBuilderFactory = DocumentBuilderFactory