diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/AllocationConfiguration.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/AllocationConfiguration.java index e48e04b486c..ac980a84400 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/AllocationConfiguration.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/AllocationConfiguration.java @@ -31,6 +31,7 @@ import org.apache.hadoop.yarn.server.resourcemanager.reservation.ReservationSchedulerConfiguration; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.allocation.AllocationFileParser; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.allocation.QueueProperties; +import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.placement.queue.QueuePlacementPolicy; import org.apache.hadoop.yarn.util.resource.Resources; import com.google.common.annotations.VisibleForTesting; diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/AllocationFileLoaderService.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/AllocationFileLoaderService.java index 3300948ce70..e15812616b2 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/AllocationFileLoaderService.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/AllocationFileLoaderService.java @@ -37,6 +37,7 @@ import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.allocation.AllocationFileParser; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.allocation.AllocationFileQueueParser; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.allocation.QueueProperties; +import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.placement.queue.QueuePlacementPolicy; import org.apache.hadoop.yarn.util.Clock; import org.apache.hadoop.yarn.util.SystemClock; import org.w3c.dom.Document; diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FairScheduler.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FairScheduler.java index 43a47ae65fe..86c88d8af1d 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FairScheduler.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FairScheduler.java @@ -91,6 +91,8 @@ import org.apache.hadoop.yarn.server.resourcemanager.scheduler.event.ReleaseContainerEvent; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.event.SchedulerEvent; +import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.placement.queue.QueueAssignmentResult; +import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.placement.queue.QueuePlacementPolicy; import org.apache.hadoop.yarn.server.resourcemanager.security.RMContainerTokenSecretManager; import org.apache.hadoop.yarn.util.resource.DefaultResourceCalculator; import org.apache.hadoop.yarn.util.resource.DominantResourceCalculator; @@ -606,10 +608,13 @@ FSLeafQueue assignToQueue(RMApp rmApp, String queueName, String user) { try { QueuePlacementPolicy placementPolicy = allocConf.getPlacementPolicy(); - queueName = placementPolicy.assignAppToQueue(queueName, user); - if (queueName == null) { - appRejectMsg = "Application rejected by queue placement policy"; + QueueAssignmentResult assignmentResult = + placementPolicy.assignAppToQueue(queueName, user); + if (assignmentResult.getType() == QueueAssignmentResult.Type.REJECTED) { + appRejectMsg = "Application rejected by queue placement policy because: " + + assignmentResult.getMessage(); } else { + queueName = assignmentResult.getQueue(); queue = queueMgr.getLeafQueue(queueName, true); if (queue == null) { appRejectMsg = queueName + " is not a leaf queue"; @@ -634,6 +639,7 @@ FSLeafQueue assignToQueue(RMApp rmApp, String queueName, String user) { } if (rmApp != null) { + assert(queue != null); rmApp.setQueue(queue.getName()); } else { LOG.error("Couldn't find RM app to set queue name on"); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FairSchedulerConfiguration.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FairSchedulerConfiguration.java index 8c4932bfe67..3e77981e533 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FairSchedulerConfiguration.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FairSchedulerConfiguration.java @@ -88,13 +88,15 @@ /** Whether pools can be created that were not specified in the FS configuration file */ - protected static final String ALLOW_UNDECLARED_POOLS = CONF_PREFIX + "allow-undeclared-pools"; - protected static final boolean DEFAULT_ALLOW_UNDECLARED_POOLS = true; + public static final String ALLOW_UNDECLARED_POOLS = + CONF_PREFIX + "allow-undeclared-pools"; + public static final boolean DEFAULT_ALLOW_UNDECLARED_POOLS = true; /** Whether to use the user name as the queue name (instead of "default") if * the request does not specify a queue. */ - protected static final String USER_AS_DEFAULT_QUEUE = CONF_PREFIX + "user-as-default-queue"; - protected static final boolean DEFAULT_USER_AS_DEFAULT_QUEUE = true; + public static final String USER_AS_DEFAULT_QUEUE = + CONF_PREFIX + "user-as-default-queue"; + public static final boolean DEFAULT_USER_AS_DEFAULT_QUEUE = true; protected static final float DEFAULT_LOCALITY_THRESHOLD = -1.0f; diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/QueuePlacementRule.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/QueuePlacementRule.java deleted file mode 100644 index 2c4add4e258..00000000000 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/QueuePlacementRule.java +++ /dev/null @@ -1,366 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair; - -import java.io.IOException; -import java.util.HashMap; -import java.util.List; -import java.util.Map; -import java.util.Set; - -import org.apache.commons.logging.Log; -import org.apache.commons.logging.LogFactory; -import org.apache.hadoop.classification.InterfaceAudience.Private; -import org.apache.hadoop.classification.InterfaceStability.Unstable; -import org.apache.hadoop.security.Groups; -import org.apache.hadoop.yarn.conf.YarnConfiguration; -import org.w3c.dom.Element; -import org.w3c.dom.NamedNodeMap; -import org.w3c.dom.Node; -import org.w3c.dom.NodeList; - -import com.google.common.annotations.VisibleForTesting; - -@Private -@Unstable -public abstract class QueuePlacementRule { - protected boolean create; - public static final Log LOG = - LogFactory.getLog(QueuePlacementRule.class.getName()); - - /** - * Initializes the rule with any arguments. - * - * @param args - * Additional attributes of the rule's xml element other than create. - */ - public QueuePlacementRule initialize(boolean create, Map args) { - this.create = create; - return this; - } - - /** - * - * @param requestedQueue - * The queue explicitly requested. - * @param user - * The user submitting the app. - * @param groups - * The groups of the user submitting the app. - * @param configuredQueues - * The queues specified in the scheduler configuration. - * @return - * The queue to place the app into. An empty string indicates that we should - * continue to the next rule, and null indicates that the app should be rejected. - */ - public String assignAppToQueue(String requestedQueue, String user, - Groups groups, Map> configuredQueues) - throws IOException { - String queue = getQueueForApp(requestedQueue, user, groups, - configuredQueues); - if (create || configuredQueues.get(FSQueueType.LEAF).contains(queue) - || configuredQueues.get(FSQueueType.PARENT).contains(queue)) { - return queue; - } else { - return ""; - } - } - - public void initializeFromXml(Element el) - throws AllocationConfigurationException { - boolean create = true; - NamedNodeMap attributes = el.getAttributes(); - Map args = new HashMap(); - for (int i = 0; i < attributes.getLength(); i++) { - Node node = attributes.item(i); - String key = node.getNodeName(); - String value = node.getNodeValue(); - if (key.equals("create")) { - create = Boolean.parseBoolean(value); - } else { - args.put(key, value); - } - } - initialize(create, args); - } - - /** - * Returns true if this rule never tells the policy to continue. - */ - public abstract boolean isTerminal(); - - /** - * Applies this rule to an app with the given requested queue and user/group - * information. - * - * @param requestedQueue - * The queue specified in the ApplicationSubmissionContext - * @param user - * The user submitting the app. - * @param groups - * The groups of the user submitting the app. - * @return - * The name of the queue to assign the app to, or null to empty string - * continue to the next rule. - */ - protected abstract String getQueueForApp(String requestedQueue, String user, - Groups groups, Map> configuredQueues) - throws IOException; - - /** - * Places apps in queues by username of the submitter - */ - public static class User extends QueuePlacementRule { - @Override - protected String getQueueForApp(String requestedQueue, String user, - Groups groups, Map> configuredQueues) { - return "root." + cleanName(user); - } - - @Override - public boolean isTerminal() { - return create; - } - } - - /** - * Places apps in queues by primary group of the submitter - */ - public static class PrimaryGroup extends QueuePlacementRule { - @Override - protected String getQueueForApp(String requestedQueue, String user, - Groups groups, Map> configuredQueues) - throws IOException { - final List groupList = groups.getGroups(user); - if (groupList.isEmpty()) { - throw new IOException("No groups returned for user " + user); - } - return "root." + cleanName(groupList.get(0)); - } - - @Override - public boolean isTerminal() { - return create; - } - } - - /** - * Places apps in queues by secondary group of the submitter - * - * Match will be made on first secondary group that exist in - * queues - */ - public static class SecondaryGroupExistingQueue extends QueuePlacementRule { - @Override - protected String getQueueForApp(String requestedQueue, String user, - Groups groups, Map> configuredQueues) - throws IOException { - List groupNames = groups.getGroups(user); - for (int i = 1; i < groupNames.size(); i++) { - String group = cleanName(groupNames.get(i)); - if (configuredQueues.get(FSQueueType.LEAF).contains("root." + group) - || configuredQueues.get(FSQueueType.PARENT).contains( - "root." + group)) { - return "root." + group; - } - } - - return ""; - } - - @Override - public boolean isTerminal() { - return false; - } - } - - /** - * Places apps in queues with name of the submitter under the queue - * returned by the nested rule. - */ - public static class NestedUserQueue extends QueuePlacementRule { - @VisibleForTesting - QueuePlacementRule nestedRule; - - /** - * Parse xml and instantiate the nested rule - */ - @Override - public void initializeFromXml(Element el) - throws AllocationConfigurationException { - NodeList elements = el.getChildNodes(); - - for (int i = 0; i < elements.getLength(); i++) { - Node node = elements.item(i); - if (node instanceof Element) { - Element element = (Element) node; - if ("rule".equals(element.getTagName())) { - QueuePlacementRule rule = QueuePlacementPolicy - .createAndInitializeRule(node); - if (rule == null) { - throw new AllocationConfigurationException( - "Unable to create nested rule in nestedUserQueue rule"); - } - this.nestedRule = rule; - break; - } else { - continue; - } - } - } - - if (this.nestedRule == null) { - throw new AllocationConfigurationException( - "No nested rule specified in rule"); - } - super.initializeFromXml(el); - } - - @Override - protected String getQueueForApp(String requestedQueue, String user, - Groups groups, Map> configuredQueues) - throws IOException { - // Apply the nested rule - String queueName = nestedRule.assignAppToQueue(requestedQueue, user, - groups, configuredQueues); - - if (queueName != null && queueName.length() != 0) { - if (!queueName.startsWith("root.")) { - queueName = "root." + queueName; - } - - // Verify if the queue returned by the nested rule is an configured leaf queue, - // if yes then skip to next rule in the queue placement policy - if (configuredQueues.get(FSQueueType.LEAF).contains(queueName)) { - return ""; - } - return queueName + "." + cleanName(user); - } - return queueName; - } - - @Override - public boolean isTerminal() { - return false; - } - } - - /** - * Places apps in queues by requested queue of the submitter - */ - public static class Specified extends QueuePlacementRule { - @Override - protected String getQueueForApp(String requestedQueue, String user, - Groups groups, Map> configuredQueues) { - if (requestedQueue.equals(YarnConfiguration.DEFAULT_QUEUE_NAME)) { - return ""; - } else { - if (!requestedQueue.startsWith("root.")) { - requestedQueue = "root." + requestedQueue; - } - return requestedQueue; - } - } - - @Override - public boolean isTerminal() { - return false; - } - } - - /** - * Places apps in the specified default queue. If no default queue is - * specified the app is placed in root.default queue. - */ - public static class Default extends QueuePlacementRule { - @VisibleForTesting - String defaultQueueName; - - @Override - public QueuePlacementRule initialize(boolean create, - Map args) { - if (defaultQueueName == null) { - defaultQueueName = "root." + YarnConfiguration.DEFAULT_QUEUE_NAME; - } - return super.initialize(create, args); - } - - @Override - public void initializeFromXml(Element el) - throws AllocationConfigurationException { - defaultQueueName = el.getAttribute("queue"); - if (defaultQueueName != null && !defaultQueueName.isEmpty()) { - if (!defaultQueueName.startsWith("root.")) { - defaultQueueName = "root." + defaultQueueName; - } - } else { - defaultQueueName = "root." + YarnConfiguration.DEFAULT_QUEUE_NAME; - } - super.initializeFromXml(el); - } - - @Override - protected String getQueueForApp(String requestedQueue, String user, - Groups groups, Map> configuredQueues) { - return defaultQueueName; - } - - @Override - public boolean isTerminal() { - return true; - } - } - - /** - * Rejects all apps - */ - public static class Reject extends QueuePlacementRule { - @Override - public String assignAppToQueue(String requestedQueue, String user, - Groups groups, Map> configuredQueues) { - return null; - } - - @Override - protected String getQueueForApp(String requestedQueue, String user, - Groups groups, Map> configuredQueues) { - throw new UnsupportedOperationException(); - } - - @Override - public boolean isTerminal() { - return true; - } - } - - /** - * Replace the periods in the username or groupname with "_dot_" and - * remove trailing and leading whitespace. - */ - protected String cleanName(String name) { - name = name.trim(); - if (name.contains(".")) { - String converted = name.replaceAll("\\.", "_dot_"); - LOG.warn("Name " + name + " is converted to " + converted - + " when it is used as a queue name."); - return converted; - } else { - return name; - } - } -} diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/placement/queue/QueueAssignmentResult.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/placement/queue/QueueAssignmentResult.java new file mode 100644 index 00000000000..75d865be0d2 --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/placement/queue/QueueAssignmentResult.java @@ -0,0 +1,74 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.placement.queue; + +import org.apache.hadoop.classification.InterfaceAudience.Private; +import org.apache.hadoop.classification.InterfaceStability.Unstable; + +/** + * The queue placement result. There are 3 types: SKIPPED, ASSIGNED, REJECTED. + * The SKIPPED result would continue on the chain of assignment rules, while + * the others would terminate the chain. + * + * The result would carry the queue if the type is ASSIGNED. + * The result would carry the reject reason if the type is REJECTED. + */ +@Private +@Unstable +public class QueueAssignmentResult { + public static final QueueAssignmentResult SKIPPED = + new QueueAssignmentResult(Type.SKIPPED, null, null); + + public static QueueAssignmentResult createAssigned(String queue) { + return new QueueAssignmentResult(Type.ASSIGNED, queue, null); + } + + public static QueueAssignmentResult createRejected(String message) { + return new QueueAssignmentResult(Type.REJECTED, null, message); + } + + private final Type type; + private final String queue; + private final String message; + + public QueueAssignmentResult(Type type, String queue, String message) { + this.type = type; + this.queue = queue; + this.message = message; + } + + public Type getType() { + return type; + } + + public String getQueue() { + return queue; + } + + public String getMessage() { + return message; + } + + /** + * The type of queue assignment result. The SKIPPED result would continue on + * the chain of assignment rules, while the others would terminate the chain. + */ + public enum Type { + ASSIGNED, SKIPPED, REJECTED + } +} diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/QueuePlacementPolicy.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/placement/queue/QueuePlacementPolicy.java similarity index 72% rename from hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/QueuePlacementPolicy.java rename to hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/placement/queue/QueuePlacementPolicy.java index 30ea213529d..4d065ee6fcb 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/QueuePlacementPolicy.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/placement/queue/QueuePlacementPolicy.java @@ -15,7 +15,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair; +package org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.placement.queue; import java.io.IOException; import java.util.ArrayList; @@ -30,6 +30,16 @@ import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.security.Groups; import org.apache.hadoop.util.ReflectionUtils; +import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.AllocationConfigurationException; +import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.FSQueueType; +import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.FairSchedulerConfiguration; +import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.placement.queue.rules.Default; +import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.placement.queue.rules.NestedUserQueue; +import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.placement.queue.rules.PrimaryGroup; +import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.placement.queue.rules.Reject; +import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.placement.queue.rules.SecondaryGroupExistingQueue; +import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.placement.queue.rules.Specified; +import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.placement.queue.rules.User; import org.w3c.dom.Element; import org.w3c.dom.Node; import org.w3c.dom.NodeList; @@ -41,40 +51,29 @@ static { Map> map = new HashMap>(); - map.put("user", QueuePlacementRule.User.class); - map.put("primaryGroup", QueuePlacementRule.PrimaryGroup.class); + map.put("user", User.class); + map.put("primaryGroup", PrimaryGroup.class); map.put("secondaryGroupExistingQueue", - QueuePlacementRule.SecondaryGroupExistingQueue.class); - map.put("specified", QueuePlacementRule.Specified.class); + SecondaryGroupExistingQueue.class); + map.put("specified", Specified.class); map.put("nestedUserQueue", - QueuePlacementRule.NestedUserQueue.class); - map.put("default", QueuePlacementRule.Default.class); - map.put("reject", QueuePlacementRule.Reject.class); + NestedUserQueue.class); + map.put("default", Default.class); + map.put("reject", Reject.class); ruleClasses = Collections.unmodifiableMap(map); } - + private final List rules; private final Map> configuredQueues; private final Groups groups; - + public QueuePlacementPolicy(List rules, - Map> configuredQueues, Configuration conf) - throws AllocationConfigurationException { - for (int i = 0; i < rules.size()-1; i++) { - if (rules.get(i).isTerminal()) { - throw new AllocationConfigurationException("Rules after rule " - + i + " in queue placement policy can never be reached"); - } - } - if (!rules.get(rules.size()-1).isTerminal()) { - throw new AllocationConfigurationException( - "Could get past last queue placement rule without assigning"); - } + Map> configuredQueues, Configuration conf) { this.rules = rules; this.configuredQueues = configuredQueues; groups = new Groups(conf); } - + /** * Builds a QueuePlacementPolicy from an xml element. */ @@ -92,7 +91,7 @@ public static QueuePlacementPolicy fromXml(Element el, } return new QueuePlacementPolicy(rules, configuredQueues, conf); } - + /** * Create and initialize a rule given a xml node * @param node @@ -118,7 +117,7 @@ public static QueuePlacementRule createAndInitializeRule(Node node) rule.initializeFromXml(element); return rule; } - + /** * Build a simple queue placement policy from the allow-undeclared-pools and * user-as-default-queue configuration options. @@ -132,25 +131,20 @@ public static QueuePlacementPolicy fromConfiguration(Configuration conf, FairSchedulerConfiguration.USER_AS_DEFAULT_QUEUE, FairSchedulerConfiguration.DEFAULT_USER_AS_DEFAULT_QUEUE); List rules = new ArrayList(); - rules.add(new QueuePlacementRule.Specified().initialize(create, null)); + rules.add(new Specified().initialize(create, null)); if (userAsDefaultQueue) { - rules.add(new QueuePlacementRule.User().initialize(create, null)); + rules.add(new User().initialize(create, null)); } if (!userAsDefaultQueue || !create) { - rules.add(new QueuePlacementRule.Default().initialize(true, null)); - } - try { - return new QueuePlacementPolicy(rules, configuredQueues, conf); - } catch (AllocationConfigurationException ex) { - throw new RuntimeException("Should never hit exception when loading" + - "placement policy from conf", ex); + rules.add(new Default().initialize(true, null)); } + return new QueuePlacementPolicy(rules, configuredQueues, conf); } /** * Applies this rule to an app with the given requested queue and user/group * information. - * + * * @param requestedQueue * The queue specified in the ApplicationSubmissionContext * @param user @@ -161,19 +155,19 @@ public static QueuePlacementPolicy fromConfiguration(Configuration conf, * @throws IOException * If an exception is encountered while getting the user's groups */ - public String assignAppToQueue(String requestedQueue, String user) - throws IOException { + public QueueAssignmentResult assignAppToQueue( + String requestedQueue, String user) throws IOException { for (QueuePlacementRule rule : rules) { - String queue = rule.assignAppToQueue(requestedQueue, user, groups, - configuredQueues); - if (queue == null || !queue.isEmpty()) { - return queue; + QueueAssignmentResult assignment = rule.assignAppToQueue( + requestedQueue, user, groups, configuredQueues); + if (assignment.getType() != QueueAssignmentResult.Type.SKIPPED) { + return assignment; } } throw new IllegalStateException("Should have applied a rule before " + - "reaching here"); + "reaching here"); } - + public List getRules() { return rules; } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/placement/queue/QueuePlacementRule.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/placement/queue/QueuePlacementRule.java new file mode 100644 index 00000000000..8a34bc867d0 --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/placement/queue/QueuePlacementRule.java @@ -0,0 +1,143 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.placement.queue; + +import java.io.IOException; +import java.util.HashMap; +import java.util.Map; +import java.util.Set; + +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.apache.hadoop.classification.InterfaceAudience.Private; +import org.apache.hadoop.classification.InterfaceStability.Unstable; +import org.apache.hadoop.security.Groups; +import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.AllocationConfigurationException; +import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.FSQueueType; +import org.w3c.dom.Element; +import org.w3c.dom.NamedNodeMap; +import org.w3c.dom.Node; + +@Private +@Unstable +public abstract class QueuePlacementRule { + protected boolean create; + public static final Log LOG = + LogFactory.getLog(QueuePlacementRule.class.getName()); + + /** + * Initializes the rule with any arguments. + * + * @param args + * Additional attributes of the rule's xml element other than create. + */ + public QueuePlacementRule initialize(boolean create, Map args) { + this.create = create; + return this; + } + + /** + * + * @param requestedQueue + * The queue explicitly requested. + * @param user + * The user submitting the app. + * @param groups + * The groups of the user submitting the app. + * @param configuredQueues + * The queues specified in the scheduler configuration. + * @return + * The queue to place the app into. An empty string indicates that we should + * continue to the next rule, and null indicates that the app should be rejected. + */ + public QueueAssignmentResult assignAppToQueue(String requestedQueue, + String user, Groups groups, + Map> configuredQueues) + throws IOException { + final QueueAssignmentResult assignment = getQueueForApp(requestedQueue, + user, groups, configuredQueues); + if (assignment.getType() == QueueAssignmentResult.Type.ASSIGNED) { + final String queue = assignment.getQueue(); + if (create || configuredQueues.get(FSQueueType.LEAF).contains(queue) + || configuredQueues.get(FSQueueType.PARENT).contains(queue)) { + return assignment; + } else { + return QueueAssignmentResult.SKIPPED; + } + } else { + return assignment; + } + } + + public void initializeFromXml(Element el) + throws AllocationConfigurationException { + boolean create = true; + NamedNodeMap attributes = el.getAttributes(); + Map args = new HashMap(); + for (int i = 0; i < attributes.getLength(); i++) { + Node node = attributes.item(i); + String key = node.getNodeName(); + String value = node.getNodeValue(); + if (key.equals("create")) { + create = Boolean.parseBoolean(value); + } else { + args.put(key, value); + } + } + initialize(create, args); + } + + public boolean isCreate() { + return create; + } + + /** + * Applies this rule to an app with the given requested queue and user/group + * information. + * + * @param requestedQueue + * The queue specified in the ApplicationSubmissionContext + * @param user + * The user submitting the app. + * @param groups + * The groups of the user submitting the app. + * @return + * The name of the queue to assign the app to, or null to empty string + * continue to the next rule. + */ + protected abstract QueueAssignmentResult getQueueForApp(String requestedQueue, + String user, Groups groups, + Map> configuredQueues) + throws IOException; + + /** + * Replace the periods in the username or groupname with "_dot_" and + * remove trailing and leading whitespace. + */ + protected String cleanName(String name) { + name = name.trim(); + if (name.contains(".")) { + String converted = name.replaceAll("\\.", "_dot_"); + LOG.warn("Name " + name + " is converted to " + converted + + " when it is used as a queue name."); + return converted; + } else { + return name; + } + } +} diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/placement/queue/rules/Default.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/placement/queue/rules/Default.java new file mode 100644 index 00000000000..58ab34636e6 --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/placement/queue/rules/Default.java @@ -0,0 +1,71 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.placement.queue.rules; + +import org.apache.hadoop.security.Groups; +import org.apache.hadoop.yarn.conf.YarnConfiguration; +import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.AllocationConfigurationException; +import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.FSQueueType; +import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.placement.queue.QueueAssignmentResult; +import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.placement.queue.QueuePlacementRule; +import org.w3c.dom.Element; + +import java.util.Map; +import java.util.Set; + +/** + * Places apps in the specified default queue. If no default queue is + * specified the app is placed in root.default queue. + */ +public class Default extends QueuePlacementRule { + private String defaultQueueName; + + @Override + public QueuePlacementRule initialize(boolean create, + Map args) { + if (defaultQueueName == null) { + defaultQueueName = "root." + YarnConfiguration.DEFAULT_QUEUE_NAME; + } + return super.initialize(create, args); + } + + @Override + public void initializeFromXml(Element el) + throws AllocationConfigurationException { + defaultQueueName = el.getAttribute("queue"); + if (defaultQueueName != null && !defaultQueueName.isEmpty()) { + if (!defaultQueueName.startsWith("root.")) { + defaultQueueName = "root." + defaultQueueName; + } + } else { + defaultQueueName = "root." + YarnConfiguration.DEFAULT_QUEUE_NAME; + } + super.initializeFromXml(el); + } + + @Override + protected QueueAssignmentResult getQueueForApp(String requestedQueue, + String user, Groups groups, + Map> configuredQueues) { + return QueueAssignmentResult.createAssigned(defaultQueueName); + } + + public String getDefaultQueueName() { + return defaultQueueName; + } +} diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/placement/queue/rules/NestedUserQueue.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/placement/queue/rules/NestedUserQueue.java new file mode 100644 index 00000000000..d932e4b2bcc --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/placement/queue/rules/NestedUserQueue.java @@ -0,0 +1,105 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.placement.queue.rules; + +import org.apache.hadoop.security.Groups; +import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.AllocationConfigurationException; +import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.FSQueueType; +import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.placement.queue.QueueAssignmentResult; +import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.placement.queue.QueuePlacementPolicy; +import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.placement.queue.QueuePlacementRule; +import org.w3c.dom.Element; +import org.w3c.dom.Node; +import org.w3c.dom.NodeList; + +import java.io.IOException; +import java.util.Map; +import java.util.Set; + +/** + * Places apps in queues with name of the submitter under the queue + * returned by the nested rule. + */ +public class NestedUserQueue extends QueuePlacementRule { + private QueuePlacementRule nestedRule; + + /** + * Parse xml and instantiate the nested rule + */ + @Override + public void initializeFromXml(Element el) + throws AllocationConfigurationException { + NodeList elements = el.getChildNodes(); + + for (int i = 0; i < elements.getLength(); i++) { + Node node = elements.item(i); + if (node instanceof Element) { + Element element = (Element) node; + if ("rule".equals(element.getTagName())) { + QueuePlacementRule rule = QueuePlacementPolicy + .createAndInitializeRule(node); + if (rule == null) { + throw new AllocationConfigurationException( + "Unable to create nested rule in nestedUserQueue rule"); + } + this.nestedRule = rule; + break; + } else { + continue; + } + } + } + + if (this.nestedRule == null) { + throw new AllocationConfigurationException( + "No nested rule specified in rule"); + } + super.initializeFromXml(el); + } + + @Override + protected QueueAssignmentResult getQueueForApp(String requestedQueue, + String user, Groups groups, + Map> configuredQueues) + throws IOException { + // Apply the nested rule + QueueAssignmentResult assignmentResult = nestedRule.assignAppToQueue( + requestedQueue, user, groups, configuredQueues); + + if (assignmentResult.getType() == QueueAssignmentResult.Type.ASSIGNED) { + String queueName = assignmentResult.getQueue(); + if (!queueName.startsWith("root.")) { + queueName = "root." + queueName; + } + + // Verify if the queue returned by the nested rule is an configured + // leaf queue, if yes then skip to next rule in the queue placement policy + if (configuredQueues.get(FSQueueType.LEAF).contains(queueName)) { + return QueueAssignmentResult.SKIPPED; + } + return QueueAssignmentResult.createAssigned( + queueName + "." + cleanName(user)); + } else { + return assignmentResult; + } + } + + public QueuePlacementRule getNestedRule() { + return nestedRule; + } +} diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/placement/queue/rules/PrimaryGroup.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/placement/queue/rules/PrimaryGroup.java new file mode 100644 index 00000000000..63fb8644553 --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/placement/queue/rules/PrimaryGroup.java @@ -0,0 +1,46 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.placement.queue.rules; + +import org.apache.hadoop.security.Groups; +import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.FSQueueType; +import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.placement.queue.QueueAssignmentResult; +import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.placement.queue.QueuePlacementRule; + +import java.io.IOException; +import java.util.List; +import java.util.Map; +import java.util.Set; + +/** + * Places apps in queues by primary group of the submitter. + */ +public class PrimaryGroup extends QueuePlacementRule { + @Override + protected QueueAssignmentResult getQueueForApp(String requestedQueue, + String user, Groups groups, + Map> configuredQueues) + throws IOException { + final List groupList = groups.getGroups(user); + if (groupList.isEmpty()) { + throw new IOException("No groups returned for user " + user); + } + return QueueAssignmentResult.createAssigned( + "root." + cleanName(groupList.get(0))); + } +} diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/placement/queue/rules/Reject.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/placement/queue/rules/Reject.java new file mode 100644 index 00000000000..157d19ef3ed --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/placement/queue/rules/Reject.java @@ -0,0 +1,45 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.placement.queue.rules; + +import org.apache.hadoop.security.Groups; +import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.FSQueueType; +import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.placement.queue.QueueAssignmentResult; +import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.placement.queue.QueuePlacementRule; + +import java.util.Map; +import java.util.Set; + +/** + * Rejects all apps. + */ +public class Reject extends QueuePlacementRule { + @Override + public QueueAssignmentResult assignAppToQueue(String requestedQueue, + String user, Groups groups, + Map> configuredQueues) { + return QueueAssignmentResult.createRejected("Final rejecting rule"); + } + + @Override + protected QueueAssignmentResult getQueueForApp(String requestedQueue, + String user, Groups groups, + Map> configuredQueues) { + throw new UnsupportedOperationException(); + } +} diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/placement/queue/rules/SecondaryGroupExistingQueue.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/placement/queue/rules/SecondaryGroupExistingQueue.java new file mode 100644 index 00000000000..e69261d37a4 --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/placement/queue/rules/SecondaryGroupExistingQueue.java @@ -0,0 +1,54 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.placement.queue.rules; + +import org.apache.hadoop.security.Groups; +import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.FSQueueType; +import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.placement.queue.QueueAssignmentResult; +import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.placement.queue.QueuePlacementRule; + +import java.io.IOException; +import java.util.List; +import java.util.Map; +import java.util.Set; + +/** + * Places apps in queues by secondary group of the submitter. + * + * Match will be made on first secondary group that exist in + * queues. + */ +public class SecondaryGroupExistingQueue extends QueuePlacementRule { + @Override + protected QueueAssignmentResult getQueueForApp(String requestedQueue, + String user, Groups groups, + Map> configuredQueues) + throws IOException { + List groupNames = groups.getGroups(user); + for (int i = 1; i < groupNames.size(); i++) { + String group = cleanName(groupNames.get(i)); + if (configuredQueues.get(FSQueueType.LEAF).contains("root." + group) + || configuredQueues.get(FSQueueType.PARENT).contains( + "root." + group)) { + return QueueAssignmentResult.createAssigned("root." + group); + } + } + + return QueueAssignmentResult.SKIPPED; + } +} diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/placement/queue/rules/Specified.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/placement/queue/rules/Specified.java new file mode 100644 index 00000000000..65758132753 --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/placement/queue/rules/Specified.java @@ -0,0 +1,46 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.placement.queue.rules; + +import org.apache.hadoop.security.Groups; +import org.apache.hadoop.yarn.conf.YarnConfiguration; +import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.FSQueueType; +import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.placement.queue.QueueAssignmentResult; +import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.placement.queue.QueuePlacementRule; + +import java.util.Map; +import java.util.Set; + +/** + * Places apps in queues by requested queue of the submitter. + */ +public class Specified extends QueuePlacementRule { + @Override + protected QueueAssignmentResult getQueueForApp(String requestedQueue, + String user, Groups groups, + Map> configuredQueues) { + if (requestedQueue.equals(YarnConfiguration.DEFAULT_QUEUE_NAME)) { + return QueueAssignmentResult.SKIPPED; + } else { + if (!requestedQueue.startsWith("root.")) { + requestedQueue = "root." + requestedQueue; + } + return QueueAssignmentResult.createAssigned(requestedQueue); + } + } +} diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/placement/queue/rules/User.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/placement/queue/rules/User.java new file mode 100644 index 00000000000..e2598ba5e5f --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/placement/queue/rules/User.java @@ -0,0 +1,38 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.placement.queue.rules; + +import org.apache.hadoop.security.Groups; +import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.FSQueueType; +import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.placement.queue.QueueAssignmentResult; +import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.placement.queue.QueuePlacementRule; + +import java.util.Map; +import java.util.Set; + +/** + * Places apps in queues by username of the submitter. + */ +public class User extends QueuePlacementRule { + @Override + protected QueueAssignmentResult getQueueForApp(String requestedQueue, + String user, Groups groups, + Map> configuredQueues) { + return QueueAssignmentResult.createAssigned("root." + cleanName(user)); + } +} diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestAllocationFileLoaderService.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestAllocationFileLoaderService.java index 50a003ecd11..dd0f6ec06a7 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestAllocationFileLoaderService.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestAllocationFileLoaderService.java @@ -25,8 +25,13 @@ import org.apache.hadoop.hdfs.MiniDFSCluster; import org.apache.hadoop.yarn.conf.YarnConfiguration; import org.apache.hadoop.yarn.server.resourcemanager.reservation.ReservationSchedulerConfiguration; -import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.QueuePlacementRule.NestedUserQueue; +import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.placement.queue.QueuePlacementPolicy; +import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.placement.queue.QueuePlacementRule; +import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.placement.queue.rules.Default; +import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.placement.queue.rules.NestedUserQueue; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.allocationfile.AllocationFileWriter; +import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.placement.queue.rules.PrimaryGroup; +import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.placement.queue.rules.Specified; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.policies.DominantResourceFairnessPolicy; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.policies.FairSharePolicy; import org.apache.hadoop.yarn.util.ControlledClock; @@ -147,7 +152,7 @@ public void testReload() throws Exception { QueuePlacementPolicy policy = allocConf.getPlacementPolicy(); List rules = policy.getRules(); assertEquals(1, rules.size()); - assertEquals(QueuePlacementRule.Default.class, rules.get(0).getClass()); + assertEquals(Default.class, rules.get(0).getClass()); assertEquals(1, allocConf.getQueueMaxApps("root.queueA")); assertEquals(2, allocConf.getConfiguredQueues().get(FSQueueType.LEAF) .size()); @@ -188,12 +193,12 @@ public void testReload() throws Exception { policy = allocConf.getPlacementPolicy(); rules = policy.getRules(); assertEquals(3, rules.size()); - assertEquals(QueuePlacementRule.Specified.class, rules.get(0).getClass()); - assertEquals(QueuePlacementRule.NestedUserQueue.class, rules.get(1) + assertEquals(Specified.class, rules.get(0).getClass()); + assertEquals(NestedUserQueue.class, rules.get(1) .getClass()); - assertEquals(QueuePlacementRule.PrimaryGroup.class, - ((NestedUserQueue) (rules.get(1))).nestedRule.getClass()); - assertEquals(QueuePlacementRule.Default.class, rules.get(2).getClass()); + assertEquals(PrimaryGroup.class, + ((NestedUserQueue) (rules.get(1))).getNestedRule().getClass()); + assertEquals(Default.class, rules.get(2).getClass()); assertEquals(3, allocConf.getQueueMaxApps("root.queueB")); assertEquals(1, allocConf.getConfiguredQueues().get(FSQueueType.LEAF) .size()); @@ -560,9 +565,9 @@ public void testSimplePlacementPolicyFromConf() throws Exception { QueuePlacementPolicy placementPolicy = allocConf.getPlacementPolicy(); List rules = placementPolicy.getRules(); assertEquals(2, rules.size()); - assertEquals(QueuePlacementRule.Specified.class, rules.get(0).getClass()); - assertEquals(false, rules.get(0).create); - assertEquals(QueuePlacementRule.Default.class, rules.get(1).getClass()); + assertEquals(Specified.class, rules.get(0).getClass()); + assertEquals(false, rules.get(0).isCreate()); + assertEquals(Default.class, rules.get(1).getClass()); } /** diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFairScheduler.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFairScheduler.java index 9120d3a6cc1..74a3e0e5be7 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFairScheduler.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFairScheduler.java @@ -114,7 +114,13 @@ import org.apache.hadoop.yarn.server.resourcemanager.scheduler.event.NodeAddedSchedulerEvent; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.event.NodeRemovedSchedulerEvent; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.event.NodeUpdateSchedulerEvent; -import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.QueuePlacementRule.Default; +import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.placement.queue.QueuePlacementPolicy; +import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.placement.queue.QueuePlacementRule; +import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.placement.queue.rules.Default; +import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.placement.queue.rules.PrimaryGroup; +import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.placement.queue.rules.SecondaryGroupExistingQueue; +import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.placement.queue.rules.Specified; +import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.placement.queue.rules.User; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.policies.DominantResourceFairnessPolicy; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.policies.FifoPolicy; import org.apache.hadoop.yarn.server.utils.BuilderUtils; @@ -1818,11 +1824,11 @@ public void testQueuePlacementWithPolicy() throws Exception { ApplicationAttemptId appId; List rules = new ArrayList(); - rules.add(new QueuePlacementRule.Specified().initialize(true, null)); - rules.add(new QueuePlacementRule.User().initialize(false, null)); - rules.add(new QueuePlacementRule.PrimaryGroup().initialize(false, null)); - rules.add(new QueuePlacementRule.SecondaryGroupExistingQueue().initialize(false, null)); - rules.add(new QueuePlacementRule.Default().initialize(true, null)); + rules.add(new Specified().initialize(true, null)); + rules.add(new User().initialize(false, null)); + rules.add(new PrimaryGroup().initialize(false, null)); + rules.add(new SecondaryGroupExistingQueue().initialize(false, null)); + rules.add(new Default().initialize(true, null)); Set queues = Sets.newHashSet("root.user1", "root.user3group", "root.user4subgroup1", "root.user4subgroup2" , "root.user5subgroup2"); Map> configuredQueues = new HashMap>(); @@ -1845,9 +1851,9 @@ public void testQueuePlacementWithPolicy() throws Exception { // test without specified as first rule rules = new ArrayList(); - rules.add(new QueuePlacementRule.User().initialize(false, null)); - rules.add(new QueuePlacementRule.Specified().initialize(true, null)); - rules.add(new QueuePlacementRule.Default().initialize(true, null)); + rules.add(new User().initialize(false, null)); + rules.add(new Specified().initialize(true, null)); + rules.add(new Default().initialize(true, null)); scheduler.getAllocationConfiguration().placementPolicy = new QueuePlacementPolicy(rules, configuredQueues, conf); appId = createSchedulingRequest(1024, "somequeue", "user1"); @@ -4488,7 +4494,7 @@ public void testDefaultRuleInitializesProperlyWhenPolicyNotConfigured() for (QueuePlacementRule rule : rules) { if (rule instanceof Default) { Default defaultRule = (Default) rule; - assertNotNull(defaultRule.defaultQueueName); + assertNotNull(defaultRule.getDefaultQueueName()); } } } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestQueuePlacementPolicy.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestQueuePlacementPolicy.java index 3fe9ce31021..12afc5e3c06 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestQueuePlacementPolicy.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestQueuePlacementPolicy.java @@ -20,10 +20,8 @@ import static org.junit.Assert.*; import java.io.IOException; -import java.util.Collections; import java.util.HashMap; import java.util.HashSet; -import java.util.List; import java.util.Map; import java.util.Set; @@ -34,6 +32,8 @@ import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.CommonConfigurationKeys; import org.apache.hadoop.security.GroupMappingServiceProvider; +import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.placement.queue.QueueAssignmentResult; +import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.placement.queue.QueuePlacementPolicy; import org.junit.Before; import org.junit.BeforeClass; import org.junit.Test; @@ -67,11 +67,11 @@ public void testSpecifiedUserPolicy() throws Exception { sb.append(""); QueuePlacementPolicy policy = parse(sb.toString()); assertEquals("root.specifiedq", - policy.assignAppToQueue("specifiedq", "someuser")); + policy.assignAppToQueue("specifiedq", "someuser").getQueue()); assertEquals("root.someuser", - policy.assignAppToQueue("default", "someuser")); + policy.assignAppToQueue("default", "someuser").getQueue()); assertEquals("root.otheruser", - policy.assignAppToQueue("default", "otheruser")); + policy.assignAppToQueue("default", "otheruser").getQueue()); } @Test @@ -85,10 +85,14 @@ public void testNoCreate() throws Exception { configuredQueues.get(FSQueueType.LEAF).add("root.someuser"); QueuePlacementPolicy policy = parse(sb.toString()); - assertEquals("root.specifiedq", policy.assignAppToQueue("specifiedq", "someuser")); - assertEquals("root.someuser", policy.assignAppToQueue("default", "someuser")); - assertEquals("root.specifiedq", policy.assignAppToQueue("specifiedq", "otheruser")); - assertEquals("root.default", policy.assignAppToQueue("default", "otheruser")); + assertEquals("root.specifiedq", + policy.assignAppToQueue("specifiedq", "someuser").getQueue()); + assertEquals("root.someuser", + policy.assignAppToQueue("default", "someuser").getQueue()); + assertEquals("root.specifiedq", + policy.assignAppToQueue("specifiedq", "otheruser").getQueue()); + assertEquals("root.default", + policy.assignAppToQueue("default", "otheruser").getQueue()); } @Test @@ -100,45 +104,9 @@ public void testSpecifiedThenReject() throws Exception { sb.append(""); QueuePlacementPolicy policy = parse(sb.toString()); assertEquals("root.specifiedq", - policy.assignAppToQueue("specifiedq", "someuser")); - assertEquals(null, policy.assignAppToQueue("default", "someuser")); - } - - @Test (expected = AllocationConfigurationException.class) - public void testOmittedTerminalRule() throws Exception { - StringBuffer sb = new StringBuffer(); - sb.append(""); - sb.append(" "); - sb.append(" "); - sb.append(""); - parse(sb.toString()); - } - - @Test (expected = AllocationConfigurationException.class) - public void testTerminalRuleInMiddle() throws Exception { - StringBuffer sb = new StringBuffer(); - sb.append(""); - sb.append(" "); - sb.append(" "); - sb.append(" "); - sb.append(""); - parse(sb.toString()); - } - - @Test - public void testTerminals() throws Exception { - // Should make it through without an exception - StringBuffer sb = new StringBuffer(); - sb.append(""); - sb.append(" "); - sb.append(" "); - sb.append(""); - QueuePlacementPolicy policy = parse(sb.toString()); - try { - policy.assignAppToQueue("root.otherdefault", "user1"); - fail("Expect exception from having default rule with create=\'false\'"); - } catch (IllegalStateException se) { - } + policy.assignAppToQueue("specifiedq", "someuser").getQueue()); + assertEquals(QueueAssignmentResult.Type.REJECTED, + policy.assignAppToQueue("default", "someuser").getType()); } @Test @@ -154,7 +122,7 @@ public void testDefaultRuleWithQueueAttribute() throws Exception { QueuePlacementPolicy policy = parse(sb.toString()); assertEquals("root.someDefaultQueue", - policy.assignAppToQueue("root.default", "user1")); + policy.assignAppToQueue("root.default", "user1").getQueue()); } @Test @@ -228,18 +196,18 @@ public void testNestedUserQueuePrimaryGroup() throws Exception { // User queue would be created under primary group queue QueuePlacementPolicy policy = parse(sb.toString()); assertEquals("root.user1group.user1", - policy.assignAppToQueue("root.default", "user1")); + policy.assignAppToQueue("root.default", "user1").getQueue()); // Other rules above and below hierarchical user queue rule should work as // usual configuredQueues.get(FSQueueType.LEAF).add("root.specifiedq"); // test if specified rule(above nestedUserQueue rule) works ok assertEquals("root.specifiedq", - policy.assignAppToQueue("root.specifiedq", "user2")); + policy.assignAppToQueue("root.specifiedq", "user2").getQueue()); // test if default rule(below nestedUserQueue rule) works configuredQueues.get(FSQueueType.LEAF).add("root.user3group"); assertEquals("root.default", - policy.assignAppToQueue("root.default", "user3")); + policy.assignAppToQueue("root.default", "user3").getQueue()); } @Test @@ -258,13 +226,13 @@ public void testNestedUserQueuePrimaryGroupNoCreate() throws Exception { // Should return root.default since primary group 'root.user1group' is not // configured assertEquals("root.default", - policy.assignAppToQueue("root.default", "user1")); + policy.assignAppToQueue("root.default", "user1").getQueue()); // Let's configure primary group and check if user queue is created configuredQueues.get(FSQueueType.PARENT).add("root.user1group"); policy = parse(sb.toString()); assertEquals("root.user1group.user1", - policy.assignAppToQueue("root.default", "user1")); + policy.assignAppToQueue("root.default", "user1").getQueue()); // Both Primary group and nestedUserQueue rule has create='false' sb = new StringBuffer(); @@ -278,7 +246,7 @@ public void testNestedUserQueuePrimaryGroupNoCreate() throws Exception { // Should return root.default since primary group and user queue for user 2 // are not configured. assertEquals("root.default", - policy.assignAppToQueue("root.default", "user2")); + policy.assignAppToQueue("root.default", "user2").getQueue()); // Now configure both primary group and the user queue for user2 configuredQueues.get(FSQueueType.PARENT).add("root.user2group"); @@ -286,7 +254,7 @@ public void testNestedUserQueuePrimaryGroupNoCreate() throws Exception { policy = parse(sb.toString()); assertEquals("root.user2group.user2", - policy.assignAppToQueue("root.default", "user2")); + policy.assignAppToQueue("root.default", "user2").getQueue()); } @Test @@ -302,14 +270,14 @@ public void testNestedUserQueueSecondaryGroup() throws Exception { QueuePlacementPolicy policy = parse(sb.toString()); // Should return root.default since secondary groups are not configured assertEquals("root.default", - policy.assignAppToQueue("root.default", "user1")); + policy.assignAppToQueue("root.default", "user1").getQueue()); // configure secondary group for user1 configuredQueues.get(FSQueueType.PARENT).add("root.user1subgroup1"); policy = parse(sb.toString()); // user queue created should be created under secondary group assertEquals("root.user1subgroup1.user1", - policy.assignAppToQueue("root.default", "user1")); + policy.assignAppToQueue("root.default", "user1").getQueue()); } @Test @@ -330,9 +298,9 @@ public void testNestedUserQueueSpecificRule() throws Exception { QueuePlacementPolicy policy = parse(sb.toString()); assertEquals("root.parent1.user1", - policy.assignAppToQueue("root.parent1", "user1")); + policy.assignAppToQueue("root.parent1", "user1").getQueue()); assertEquals("root.parent2.user2", - policy.assignAppToQueue("root.parent2", "user2")); + policy.assignAppToQueue("root.parent2", "user2").getQueue()); } @Test @@ -351,7 +319,7 @@ public void testNestedUserQueueDefaultRule() throws Exception { QueuePlacementPolicy policy = parse(sb.toString()); assertEquals("root.parentq.user1", - policy.assignAppToQueue("root.default", "user1")); + policy.assignAppToQueue("root.default", "user1").getQueue()); } @Test @@ -363,7 +331,7 @@ public void testUserContainsPeriod() throws Exception { sb.append(""); QueuePlacementPolicy policy = parse(sb.toString()); assertEquals("root.first_dot_last", - policy.assignAppToQueue("default", "first.last")); + policy.assignAppToQueue("default", "first.last").getQueue()); sb = new StringBuffer(); sb.append(""); @@ -375,7 +343,7 @@ public void testUserContainsPeriod() throws Exception { sb.append(""); policy = parse(sb.toString()); assertEquals("root.default.first_dot_last", - policy.assignAppToQueue("root.default", "first.last")); + policy.assignAppToQueue("root.default", "first.last").getQueue()); } @Test @@ -395,7 +363,7 @@ public void testGroupContainsPeriod() throws Exception { // in the group name should be converted into _dot_ QueuePlacementPolicy policy = parse(sb.toString()); assertEquals("root.user1_dot_group.user1", - policy.assignAppToQueue("root.default", "user1")); + policy.assignAppToQueue("root.default", "user1").getQueue()); conf.setClass(CommonConfigurationKeys.HADOOP_SECURITY_GROUP_MAPPING, SimpleGroupsMapping.class, GroupMappingServiceProvider.class);