diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMAppManager.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMAppManager.java index 783fab0bc05..2da2249bdab 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMAppManager.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMAppManager.java @@ -68,6 +68,8 @@ import org.apache.hadoop.yarn.server.resourcemanager.scheduler.YarnScheduler; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CSQueue; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacityScheduler; +import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.FSQueue; +import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.FairScheduler; import org.apache.hadoop.yarn.server.security.ApplicationACLsManager; import org.apache.hadoop.yarn.server.utils.BuilderUtils; import org.apache.hadoop.yarn.util.Times; @@ -412,7 +414,8 @@ private RMAppImpl createAndPopulateNewRMApp( // We only replace the queue when it's a new application if (!isRecovery) { - replaceQueueFromPlacementContext(placementContext, submissionContext); + copyPlacementQueueToSubmissionContext(placementContext, + submissionContext); // fail the submission if configured application timeout value is invalid RMServerUtils.validateApplicationTimeouts( @@ -437,38 +440,59 @@ private RMAppImpl createAndPopulateNewRMApp( submissionContext.setPriority(appPriority); } - // Since FairScheduler queue mapping is done inside scheduler, - // if FairScheduler is used and the queue doesn't exist, we should not - // fail here because queue will be created inside FS. Ideally, FS queue - // mapping should be done outside scheduler too like CS. - // For now, exclude FS for the acl check. - if (!isRecovery && YarnConfiguration.isAclEnabled(conf) - && scheduler instanceof CapacityScheduler) { - String queueName = submissionContext.getQueue(); - String appName = submissionContext.getApplicationName(); - CSQueue csqueue = ((CapacityScheduler) scheduler).getQueue(queueName); - - if (csqueue == null && placementContext != null) { - //could be an auto created queue through queue mapping. Validate - // parent queue exists and has valid acls - String parentQueueName = placementContext.getParentQueue(); - csqueue = ((CapacityScheduler) scheduler).getQueue(parentQueueName); - } + if (!isRecovery && YarnConfiguration.isAclEnabled(conf)) { + if (scheduler instanceof CapacityScheduler) { + String queueName = submissionContext.getQueue(); + String appName = submissionContext.getApplicationName(); + CSQueue csqueue = ((CapacityScheduler) scheduler).getQueue(queueName); + + if (csqueue == null && placementContext != null) { + //could be an auto created queue through queue mapping. Validate + // parent queue exists and has valid acls + String parentQueueName = placementContext.getParentQueue(); + csqueue = ((CapacityScheduler) scheduler).getQueue(parentQueueName); + } - if (csqueue != null - && !authorizer.checkPermission( - new AccessRequest(csqueue.getPrivilegedEntity(), userUgi, - SchedulerUtils.toAccessType(QueueACL.SUBMIT_APPLICATIONS), - applicationId.toString(), appName, Server.getRemoteAddress(), - null)) - && !authorizer.checkPermission( - new AccessRequest(csqueue.getPrivilegedEntity(), userUgi, - SchedulerUtils.toAccessType(QueueACL.ADMINISTER_QUEUE), - applicationId.toString(), appName, Server.getRemoteAddress(), - null))) { - throw RPCUtil.getRemoteException(new AccessControlException( - "User " + user + " does not have permission to submit " - + applicationId + " to queue " + submissionContext.getQueue())); + if (csqueue != null + && !authorizer.checkPermission( + new AccessRequest(csqueue.getPrivilegedEntity(), userUgi, + SchedulerUtils.toAccessType(QueueACL.SUBMIT_APPLICATIONS), + applicationId.toString(), appName, Server.getRemoteAddress(), + null)) + && !authorizer.checkPermission( + new AccessRequest(csqueue.getPrivilegedEntity(), userUgi, + SchedulerUtils.toAccessType(QueueACL.ADMINISTER_QUEUE), + applicationId.toString(), appName, Server.getRemoteAddress(), + null))) { + throw RPCUtil.getRemoteException(new AccessControlException( + "User " + user + " does not have permission to submit " + + applicationId + " to queue " + submissionContext.getQueue())); + } + } + if (scheduler instanceof FairScheduler) { + // if we have not placed the app just skip this, the submit will be + // rejected in the scheduler. + if (placementContext != null) { + // The queue might not be created yet. Walk up the tree to check the + // parent ACL. The queueName is assured root which always exists + String queueName = submissionContext.getQueue(); + FSQueue queue = ((FairScheduler) scheduler).getQueueManager(). + getQueue(queueName); + while (queue == null) { + int sepIndex = queueName.lastIndexOf("."); + queueName = queueName.substring(0, sepIndex); + queue = ((FairScheduler) scheduler).getQueueManager(). + getQueue(queueName); + } + if (!queue.hasAccess(QueueACL.SUBMIT_APPLICATIONS, userUgi) && + !queue.hasAccess(QueueACL.ADMINISTER_QUEUE, userUgi)) { + throw RPCUtil.getRemoteException(new AccessControlException( + "User " + user + " does not have permission to submit " + + applicationId + " to queue " + + submissionContext.getQueue() + + " denied by ACL for queue " + queueName)); + } + } } } @@ -835,34 +859,39 @@ ApplicationPlacementContext placeApplication( // Placement could also fail if the user doesn't exist in system // skip if the user is not found during recovery. if (isRecovery) { - LOG.warn("PlaceApplication failed,skipping on recovery of rm"); + LOG.warn("Application placement failed for user " + user + + " and application " + context.getApplicationId() + + ", skipping placement on recovery of rm"); + LOG.debug("Exception that caused the placement failure", e); return placementContext; } throw e; } } - if (placementContext == null && (context.getQueue() == null) || context - .getQueue().isEmpty()) { + // The submission context when created often has a queue set. In case of + // the FairScheduler a null placement context is still considered as a + // failure, even when a queue is provided on submit. This case handled in + // the scheduler. + if (placementContext == null && (context.getQueue() == null) || + context.getQueue().isEmpty()) { String msg = "Failed to place application " + context.getApplicationId() - + " to queue and specified " + "queue is invalid : " + context - .getQueue(); + + " in a queue and submit context queue is null or empty"; LOG.error(msg); throw new YarnException(msg); } return placementContext; } - void replaceQueueFromPlacementContext( + private void copyPlacementQueueToSubmissionContext( ApplicationPlacementContext placementContext, ApplicationSubmissionContext context) { - // Set it to ApplicationSubmissionContext - //apply queue mapping only to new application submissions + // Set the queue from the placement in the ApplicationSubmissionContext + // Placement rule are only considered for new applications if (placementContext != null && !StringUtils.equalsIgnoreCase( context.getQueue(), placementContext.getQueue())) { - LOG.info("Placed application=" + context.getApplicationId() + - " to queue=" + placementContext.getQueue() + ", original queue=" - + context - .getQueue()); + LOG.info("Placed application with ID " + context.getApplicationId() + + " in queue: " + placementContext.getQueue() + + ", original submission queue was: " + context.getQueue()); context.setQueue(placementContext.getQueue()); } } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/placement/DefaultPlacementRule.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/placement/DefaultPlacementRule.java new file mode 100644 index 00000000000..8048021883b --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/placement/DefaultPlacementRule.java @@ -0,0 +1,126 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.yarn.server.resourcemanager.placement; + +import com.google.common.annotations.VisibleForTesting; +import org.apache.hadoop.yarn.api.records.ApplicationSubmissionContext; +import org.apache.hadoop.yarn.conf.YarnConfiguration; +import org.apache.hadoop.yarn.server.resourcemanager.scheduler.ResourceScheduler; +import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.FSQueue; +import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.FairScheduler; +import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.QueueManager; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.w3c.dom.Element; + +import java.io.IOException; + +import static org.apache.hadoop.yarn.server.resourcemanager.placement.FairQueuePlacementUtils.assureRoot; +import static org.apache.hadoop.yarn.server.resourcemanager.placement.FairQueuePlacementUtils.isValidQueueName; + +/** + * Places apps in the specified default queue. If no default queue is + * specified the app is placed in root.default queue. + */ +public class DefaultPlacementRule extends PlacementRule { + private static final Logger LOG = + LoggerFactory.getLogger(DefaultPlacementRule.class); + + private QueueManager queueManager; + @VisibleForTesting + public String defaultQueueName; + + /** + * Create a new rule from the xml config. + * @param conf An xml element from the {@link FairScheduler#conf} + */ + public DefaultPlacementRule(Element conf) { + // Default create for this rule is true + createQueue = true; + // No config can be set when no policy is defined and we use defaults + if (conf != null) { + String create = conf.getAttribute("create"); + if (create != null && !create.isEmpty()) { + createQueue = Boolean.parseBoolean(create); + } + defaultQueueName = conf.getAttribute("queue"); + } + // The queue name does not have to be set and we really use "default" + if (defaultQueueName == null || defaultQueueName.isEmpty()) { + defaultQueueName = assureRoot(YarnConfiguration.DEFAULT_QUEUE_NAME); + } else { + defaultQueueName = assureRoot(defaultQueueName); + } + LOG.debug("Default rule instantiated with queue name: {}, " + + "and create flag: {}", defaultQueueName, createQueue); + } + + /** + * Create a new rule just setting the create flag. + * @param create String form of the boolean flag, ignored by the rule. + */ + public DefaultPlacementRule(boolean create) { + createQueue = create; + // No config so fall back to the real default. + defaultQueueName = assureRoot(YarnConfiguration.DEFAULT_QUEUE_NAME); + LOG.debug("Default rule instantiated with default queue name: {}, " + + "and create flag: {}", defaultQueueName, createQueue); + } + + @Override + public boolean initialize(ResourceScheduler scheduler) throws IOException { + if (!(scheduler instanceof FairScheduler)) { + LOG.error("Default rule configured for wrong scheduler type."); + throw new IOException( + "Default rule can only be configured for the FairScheduler"); + } + // A queue read from the config could be illegal check it + if (!isValidQueueName(defaultQueueName)) { + throw new IOException( + "Default rule configured with an illegal queue name: '" + + defaultQueueName + "'."); + } + FairScheduler fs = (FairScheduler) scheduler; + queueManager = fs.getQueueManager(); + + return true; + } + + @Override + public ApplicationPlacementContext getPlacementForApp( + ApplicationSubmissionContext asc, String user) { + + // If we can create the queue in the rule or the queue exists return it + if (createQueue || configuredQueue(defaultQueueName)) { + return new ApplicationPlacementContext(defaultQueueName); + } + return null; + } + + /** + * Check if the queue exists and is part of the configuration i.e. not + * a {@link FSQueue#isDynamic()} queue. + * @param queueName name of the queue to check + * @return true if the queue exists and is a "configured" queue + */ + private boolean configuredQueue(String queueName) { + FSQueue queue = queueManager.getQueue(queueName); + return (queue != null && !queue.isDynamic()); + } +} diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/placement/FairQueuePlacementUtils.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/placement/FairQueuePlacementUtils.java new file mode 100644 index 00000000000..75792e5382c --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/placement/FairQueuePlacementUtils.java @@ -0,0 +1,92 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.yarn.server.resourcemanager.placement; + +import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.FairSchedulerUtilities; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +/** + * Utility methods used by Fair scheduler placement rules. + * {@link org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.FairScheduler} + */ +public class FairQueuePlacementUtils { + private static final Logger LOG = + LoggerFactory.getLogger(QueuePlacementRuleUtils.class); + + private static final String ROOT_QUEUE = "root"; + // Constants for name clean up and hierarchy checks + private static final String DOT_REPLACEMENT = "_dot_"; + protected static final String DOT = "."; + + /** + * Replace the periods in the username or group name with "_dot_" and + * remove trailing and leading whitespace. + * + * @param name The name to clean + * @return The name with {@link #DOT} replaced with {@link #DOT_REPLACEMENT} + */ + protected static String cleanName(String name) { + name = FairSchedulerUtilities.trimQueueName(name); + if (name.contains(DOT)) { + String converted = name.replaceAll("\\.", DOT_REPLACEMENT); + LOG.warn("Name {} is converted to {} when it is used as a queue name.", + name, converted); + return converted; + } else { + return name; + } + } + + /** + * Assure root prefix for a queue name. + * + * @param queueName The queue name to check for the root prefix + * @return The root prefixed queue name + */ + protected static String assureRoot(String queueName) { + if (queueName != null && !queueName.isEmpty()) { + if (!queueName.startsWith(ROOT_QUEUE + DOT) && + !queueName.equals(ROOT_QUEUE)) { + queueName = ROOT_QUEUE + DOT + queueName; + } + } else { + LOG.warn("AssureRoot: queueName is empty or null."); + } + return queueName; + } + + /** + * Validate the queue name: it may not start or end with a {@link #DOT}. + * + * @param queueName The queue name to validate + * @return false if the queue name starts or ends with a + * {@link #DOT}, true + */ + protected static boolean isValidQueueName(String queueName) { + if (queueName != null) { + if (queueName.equals(FairSchedulerUtilities.trimQueueName(queueName)) && + !queueName.startsWith(DOT) && + !queueName.endsWith(DOT)) { + return true; + } + } + return false; + } +} diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/placement/PlacementFactory.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/placement/PlacementFactory.java index a5de66463bb..76177120365 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/placement/PlacementFactory.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/placement/PlacementFactory.java @@ -23,6 +23,8 @@ import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.util.ReflectionUtils; +import java.lang.reflect.Constructor; + /** * Factory class for creating instances of {@link PlacementRule}. */ @@ -42,4 +44,27 @@ public static PlacementRule getPlacementRule(String ruleStr, LOG.info("Using PlacementRule implementation - " + ruleClass); return ReflectionUtils.newInstance(ruleClass, conf); } -} \ No newline at end of file + + /** + * Create a new {@link PlacementRule} based on the rule class from the + * configuration. This is used to instantiate rules with different + * constructors by the FairScheduler. + * @param A class that represent a {@link PlacementRule} + * @param theClass The specific class reference to instantiate + * @param classType argument type of the constructor + * @param initArg argument for the constructor + * @return Created class instance + */ + public static T getPlacementRule(Class theClass, Class classType, + Object initArg) { + T result; + try { + Constructor method = theClass.getDeclaredConstructor(classType); + method.setAccessible(true); + result = method.newInstance(initArg); + } catch (Exception e) { + throw new RuntimeException(e); + } + return result; + } +} diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/placement/PlacementRule.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/placement/PlacementRule.java index 0f3d43c5ad7..ae830ed0c2e 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/placement/PlacementRule.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/placement/PlacementRule.java @@ -20,11 +20,48 @@ import java.io.IOException; +import com.google.common.annotations.VisibleForTesting; import org.apache.hadoop.yarn.api.records.ApplicationSubmissionContext; import org.apache.hadoop.yarn.exceptions.YarnException; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.ResourceScheduler; public abstract class PlacementRule { + // Flag to show if the rule can create a queue + @VisibleForTesting + public boolean createQueue = false; + PlacementRule parentRule = null; + + /** + * Flag that shows if the rule must be the last rule in a series of rules. + * The rule must be the last if it always returns a queue and never + * null. + * + * @return true is the rule must be the last rule in a series + * false otherwise. + */ + public boolean isTerminal() { + if (parentRule == null) { + return createQueue; + } else { + return parentRule.isTerminal() && createQueue; + } + } + + /** + * Set a rule to generate the parent queue dynamically. + * @param parentRule A PlacmentRule + */ + public void setParentRule(PlacementRule parentRule) { + this.parentRule = parentRule; + } + + /** + * Get the rule that is set to generate the parent queue dynamically. + * @return The rule set or null if not set. + */ + public PlacementRule getParentRule() { + return parentRule; + } public String getName() { return this.getClass().getName(); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/placement/PrimaryGroupPlacementRule.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/placement/PrimaryGroupPlacementRule.java new file mode 100644 index 00000000000..922032b9b89 --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/placement/PrimaryGroupPlacementRule.java @@ -0,0 +1,146 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.yarn.server.resourcemanager.placement; + +import org.apache.hadoop.security.Groups; +import org.apache.hadoop.yarn.api.records.ApplicationSubmissionContext; +import org.apache.hadoop.yarn.exceptions.YarnException; +import org.apache.hadoop.yarn.server.resourcemanager.scheduler.ResourceScheduler; +import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.FSLeafQueue; +import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.FSQueue; +import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.FairScheduler; +import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.QueueManager; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.w3c.dom.Element; + +import java.io.IOException; +import java.util.List; + +import static org.apache.hadoop.yarn.server.resourcemanager.placement.FairQueuePlacementUtils.DOT; +import static org.apache.hadoop.yarn.server.resourcemanager.placement.FairQueuePlacementUtils.assureRoot; +import static org.apache.hadoop.yarn.server.resourcemanager.placement.FairQueuePlacementUtils.cleanName; + +/** + * Places apps in queues by the primary group of the submitter. + */ +public class PrimaryGroupPlacementRule extends PlacementRule { + private static final Logger LOG = + LoggerFactory.getLogger(PrimaryGroupPlacementRule.class); + + private QueueManager queueManager; + private Groups groupProvider; + + /** + * Create a new rule from the xml config. + * @param conf An xml element from the {@link FairScheduler#conf} + */ + public PrimaryGroupPlacementRule(Element conf) { + // No config can be set when no policy is defined and we use defaults + createQueue = true; + if (conf != null) { + String create = conf.getAttribute("create"); + if (create != null && !create.isEmpty()) { + createQueue = Boolean.parseBoolean(create); + } + } + LOG.debug("PrimaryGroupPlacementRule instantiated with create flag: {}", + createQueue); + } + + /** + * Create a new rule just setting the create flag. + * @param create String form of the boolean flag. Only true + * will set the flag to True + */ + public PrimaryGroupPlacementRule(boolean create) { + createQueue = create; + LOG.debug("PrimaryGroupPlacementRule instantiated with create flag: {}", + createQueue); + } + + @Override + public boolean initialize(ResourceScheduler scheduler) throws IOException { + if (!(scheduler instanceof FairScheduler)) { + LOG.error("PrimaryGroup rule configured for wrong scheduler type."); + throw new IOException( + "PrimaryGroup rule can only be configured for the FairScheduler"); + } + + FairScheduler fs = (FairScheduler) scheduler; + queueManager = fs.getQueueManager(); + groupProvider = new Groups(fs.getConfig()); + if (parentRule != null && parentRule.getName().equals(getName())) { + throw new IOException( + "Parent rule should not be the same type as the child rule (PrimaryGroup)"); + } + + return true; + } + + @Override + public ApplicationPlacementContext getPlacementForApp( + ApplicationSubmissionContext asc, String user) throws YarnException { + + // All users should have at least one group the primary group. If no groups + // are returned then there is a real issue. + final List groupList; + try { + groupList = groupProvider.getGroups(user); + } catch (IOException ioe) { + throw new YarnException("Group resolution failed", ioe); + } + if (groupList.isEmpty()) { + LOG.error("Group placement rule failed: No groups returned for user {}", + user); + throw new YarnException("No groups returned for user " + user); + } + + String cleanGroup = cleanName(groupList.get(0)); + String queueName; + if (parentRule != null) { + ApplicationPlacementContext parent = + parentRule.getPlacementForApp(asc, user); + if (parent == null || + queueManager.getQueue(parent.getQueue()) instanceof FSLeafQueue) { + return null; + } + queueName = parent.getQueue() + DOT + cleanGroup; + } else { + queueName = assureRoot(cleanGroup); + } + + // If we can create the queue in the rule or the queue exists return it + if (createQueue || configuredQueue(queueName)) { + return new ApplicationPlacementContext(queueName); + } + return null; + } + + /** + * Check if the queue exists and is part of the configuration i.e. not + * a {@link FSQueue#isDynamic()} queue. + * @param queueName name of the queue to check + * @return true if the queue exists and is a "configured" queue + */ + private boolean configuredQueue(String queueName) { + FSQueue queue = queueManager.getQueue(queueName); + return (queue != null && !queue.isDynamic()); + } +} diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/placement/RejectPlacementRule.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/placement/RejectPlacementRule.java new file mode 100644 index 00000000000..f60e6673b69 --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/placement/RejectPlacementRule.java @@ -0,0 +1,86 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.yarn.server.resourcemanager.placement; + +import org.apache.hadoop.yarn.api.records.ApplicationSubmissionContext; +import org.apache.hadoop.yarn.server.resourcemanager.scheduler.ResourceScheduler; +import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.FairScheduler; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.w3c.dom.Element; + +import java.io.IOException; + +/** + * Places apps in queues by username of the submitter. + */ +public class RejectPlacementRule extends PlacementRule { + private static final Logger LOG = + LoggerFactory.getLogger(RejectPlacementRule.class); + + /** + * Create a new rule from the xml config. All placements are + * rejected and the parameters are ignored. + * @param conf An xml element from the {@link FairScheduler#conf} + */ + public RejectPlacementRule(Element conf) { + LOG.debug("RejectPlacementRule instantiated"); + } + + /** + * Create a new rule just setting the create flag. All placements are + * rejected and the parameters are ignored. + * @param create Create flag (ignored in this rule) + */ + public RejectPlacementRule(boolean create) { + LOG.debug("RejectPlacementRule instantiated"); + } + + /** + * RejectPlacementRule must always be the terminal rule, the create flag is + * ignored. + * @return true + */ + @Override + public boolean isTerminal() { + return true; + } + + @Override + public boolean initialize(ResourceScheduler scheduler) throws IOException { + if (!(scheduler instanceof FairScheduler)) { + LOG.error("Reject rule configured for wrong scheduler type."); + throw new IOException( + "Reject rule can only be configured for the FairScheduler"); + } + if (parentRule != null) { + LOG.error("You cannot configure any parent rule for the Reject rule."); + throw new IOException( + "Parent rule should not be configured for reject rule."); + } + + return true; + } + + @Override + public ApplicationPlacementContext getPlacementForApp( + ApplicationSubmissionContext asc, String user) { + return null; + } +} diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/placement/SecondaryGroupExistingPlacementRule.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/placement/SecondaryGroupExistingPlacementRule.java new file mode 100644 index 00000000000..904081166a6 --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/placement/SecondaryGroupExistingPlacementRule.java @@ -0,0 +1,159 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.yarn.server.resourcemanager.placement; + +import org.apache.hadoop.security.Groups; +import org.apache.hadoop.yarn.api.records.ApplicationSubmissionContext; +import org.apache.hadoop.yarn.exceptions.YarnException; +import org.apache.hadoop.yarn.server.resourcemanager.scheduler.ResourceScheduler; +import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.FSLeafQueue; +import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.FSQueue; +import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.FairScheduler; +import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.QueueManager; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.w3c.dom.Element; + +import java.io.IOException; +import java.util.List; + +import static org.apache.hadoop.yarn.server.resourcemanager.placement.FairQueuePlacementUtils.DOT; +import static org.apache.hadoop.yarn.server.resourcemanager.placement.FairQueuePlacementUtils.assureRoot; +import static org.apache.hadoop.yarn.server.resourcemanager.placement.FairQueuePlacementUtils.cleanName; + +/** + * Places apps in queues by the secondary group of the submitter, if the + * submitter is a member of more than one group. + * The first "matching" queue based on the group list is returned. The match + * takes into account the parent rule and create flag, + */ +public class SecondaryGroupExistingPlacementRule extends PlacementRule { + private static final Logger LOG = + LoggerFactory.getLogger(SecondaryGroupExistingPlacementRule.class); + + private QueueManager queueManager; + private Groups groupProvider; + + /** + * Create a new rule from the xml config. + * @param conf An xml element from the {@link FairScheduler#conf} + */ + public SecondaryGroupExistingPlacementRule(Element conf) { + // No config can be set when no policy is defined and we use defaults + createQueue = true; + if (conf != null) { + String create = conf.getAttribute("create"); + if (create != null && !create.isEmpty()) { + createQueue = Boolean.parseBoolean(create); + } + } + LOG.debug("SecondaryGroupExistingPlacementRule instantiated with create flag: {}", + createQueue); + } + + /** + * Create a new rule just setting the create flag. + * @param create String form of the boolean flag. Only true + * will set the flag to True + */ + public SecondaryGroupExistingPlacementRule(boolean create) { + createQueue = create; + LOG.debug("SecondaryGroupExistingPlacementRule instantiated with create " + + "flag: {}", createQueue); + } + + /** + * SecondaryGroupPlacementRule cannot be a terminal rule even if the create + * flag is true. There is no guarantee that a user is a member + * of a second group. + * @return false + */ + @Override + public boolean isTerminal() { + return false; + } + + @Override + public boolean initialize(ResourceScheduler scheduler) throws IOException { + if (!(scheduler instanceof FairScheduler)) { + LOG.error( + "SecondaryGroupExisting rule configured for wrong scheduler type."); + throw new IOException("SecondaryGroupExisting rule can only be " + "" + + "configured for the FairScheduler"); + } + + FairScheduler fs = (FairScheduler) scheduler; + queueManager = fs.getQueueManager(); + groupProvider = new Groups(fs.getConfig()); + if (parentRule != null && parentRule.getName().equals(getName())) { + LOG.error("SecondaryGroupExisting rule has parent rule of the same " + + "type configured. You cannot configure parent and child rule of " + + "the same type."); + throw new IOException("Parent rule should not be the same type as the " + + "child rule (SecondaryGroupExisting)"); + } + return true; + } + + @Override + public ApplicationPlacementContext getPlacementForApp( + ApplicationSubmissionContext asc, String user) throws YarnException { + + // All users should have at least one group the primary group. If no groups + // are returned then there is a real issue. + final List groupList; + try { + groupList = groupProvider.getGroups(user); + } catch (IOException ioe) { + throw new YarnException("Group resolution failed", ioe); + } + // first get the parent queue if needed + String parentQueue = null; + if (parentRule != null) { + ApplicationPlacementContext parent = + parentRule.getPlacementForApp(asc, user); + if (parent == null || + queueManager.getQueue(parent.getQueue()) instanceof FSLeafQueue) { + return null; + } + parentQueue = parent.getQueue(); + } + // now check the groups inside the parent + for (int i = 1; i < groupList.size(); i++) { + String group = cleanName(groupList.get(i)); + String queueName = + parentQueue == null ? assureRoot(group) : parentQueue + DOT + group; + if (configuredQueue(queueName)) { + return new ApplicationPlacementContext(queueName); + } + } + return null; + } + + /** + * Check if the queue exists and is part of the configuration i.e. not + * a {@link FSQueue#isDynamic()} queue. + * @param queueName name of the queue to check + * @return true if the queue exists and is a "configured" queue + */ + private boolean configuredQueue(String queueName) { + FSQueue queue = queueManager.getQueue(queueName); + return (queue != null && !queue.isDynamic()); + } +} diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/placement/SpecifiedPlacementRule.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/placement/SpecifiedPlacementRule.java new file mode 100644 index 00000000000..f647a361866 --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/placement/SpecifiedPlacementRule.java @@ -0,0 +1,134 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.yarn.server.resourcemanager.placement; + +import org.apache.hadoop.yarn.api.records.ApplicationSubmissionContext; +import org.apache.hadoop.yarn.conf.YarnConfiguration; +import org.apache.hadoop.yarn.exceptions.YarnException; +import org.apache.hadoop.yarn.server.resourcemanager.scheduler.ResourceScheduler; +import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.FSQueue; +import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.FairScheduler; +import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.QueueManager; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.w3c.dom.Element; + +import java.io.IOException; + +import static org.apache.hadoop.yarn.server.resourcemanager.placement.FairQueuePlacementUtils.assureRoot; +import static org.apache.hadoop.yarn.server.resourcemanager.placement.FairQueuePlacementUtils.isValidQueueName; + +/** + * Places apps in queues by requested queue of the submitter + */ +public class SpecifiedPlacementRule extends PlacementRule { + private static final Logger LOG = + LoggerFactory.getLogger(SpecifiedPlacementRule.class); + + private QueueManager queueManager; + + /** + * Create a new rule from the xml config. + * @param conf An xml element from the {@link FairScheduler#conf} + */ + public SpecifiedPlacementRule(Element conf) { + createQueue = true; + // No config can be set when no policy is defined and we use defaults + if (conf != null) { + String create = conf.getAttribute("create"); + if (create != null && !create.isEmpty()) { + createQueue = Boolean.parseBoolean(create); + } + } + LOG.debug("Specified rule instantiated with create flag: {}", + createQueue); + } + + /** + * Create a new rule just setting the create flag. + * @param create String form of the boolean flag. Only true + * will set the flag to True + */ + public SpecifiedPlacementRule(boolean create) { + createQueue = create; + LOG.debug("Specified rule instantiated with create flag: {}", + createQueue); + } + + /** + * SpecifiedPlacementRule cannot be a terminal rule even if the create flag + * is true. The "default" queue is always ignored in this rule. + * @return false + */ + @Override + public boolean isTerminal() { + return false; + } + + @Override + public boolean initialize(ResourceScheduler scheduler) throws IOException { + if (!(scheduler instanceof FairScheduler)) { + LOG.error("Specified rule configured for wrong scheduler type."); + throw new IOException( + "Specified rule can only be configured for the FairScheduler"); + } + + FairScheduler fs = (FairScheduler) scheduler; + queueManager = fs.getQueueManager(); + + return true; + } + + @Override + public ApplicationPlacementContext getPlacementForApp( + ApplicationSubmissionContext asc, String user) throws YarnException { + + // Sanity check the provided queue + String queueName = asc.getQueue(); + if (!isValidQueueName(queueName)) { + LOG.error("Specified queue name not valid: '{}'", queueName); + throw new YarnException("Application submitted by user " + user + + "with illegal queue name '" + queueName + "'."); + } + // On submission the requested queue will be set to "default" if no queue + // is specified: just check the next rule in that case + // we need to filter out the "default" queue in both forms + if (queueName.equals(YarnConfiguration.DEFAULT_QUEUE_NAME) || + queueName.equals(assureRoot(YarnConfiguration.DEFAULT_QUEUE_NAME))) { + return null; + } + queueName = assureRoot(queueName); + // If we can create the queue in the rule or the queue exists return it + if (createQueue || configuredQueue(queueName)) { + return new ApplicationPlacementContext(queueName); + } + return null; + } + + /** + * Check if the queue exists and is part of the configuration i.e. not + * a {@link FSQueue#isDynamic()} queue. + * @param queueName name of the queue to check + * @return true if the queue exists and is a "configured" queue + */ + private boolean configuredQueue(String queueName) { + FSQueue queue = queueManager.getQueue(queueName); + return (queue != null && !queue.isDynamic()); + } +} diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/placement/UserPlacementRule.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/placement/UserPlacementRule.java new file mode 100644 index 00000000000..d6ac30ed4e8 --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/placement/UserPlacementRule.java @@ -0,0 +1,126 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.yarn.server.resourcemanager.placement; + +import org.apache.hadoop.yarn.api.records.ApplicationSubmissionContext; +import org.apache.hadoop.yarn.exceptions.YarnException; +import org.apache.hadoop.yarn.server.resourcemanager.scheduler.ResourceScheduler; +import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.*; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.w3c.dom.Element; + +import java.io.IOException; + +import static org.apache.hadoop.yarn.server.resourcemanager.placement.FairQueuePlacementUtils.DOT; +import static org.apache.hadoop.yarn.server.resourcemanager.placement.FairQueuePlacementUtils.assureRoot; +import static org.apache.hadoop.yarn.server.resourcemanager.placement.FairQueuePlacementUtils.cleanName; + +/** + * Places apps in queues by username of the submitter. + */ +public class UserPlacementRule extends PlacementRule { + private static final Logger LOG = + LoggerFactory.getLogger(UserPlacementRule.class); + + private QueueManager queueManager; + + /** + * Create a new rule from the xml config. + * @param conf An xml element from the {@link FairScheduler#conf} + */ + public UserPlacementRule(Element conf) { + // No config can be set when no policy is defined and we use defaults + createQueue = true; + if (conf != null) { + String create = conf.getAttribute("create"); + if (create != null && !create.isEmpty()) { + createQueue = Boolean.parseBoolean(create); + } + } + LOG.debug("User rule instantiated with create flag: {}", + createQueue); + } + + /** + * Create a new rule just setting the create flag. + * @param create String form of the boolean flag. Only true + * will set the flag to True + */ + public UserPlacementRule(boolean create) { + createQueue = create; + LOG.debug("User rule instantiated with create flag: {}", + createQueue); + } + + @Override + public boolean initialize(ResourceScheduler scheduler) throws IOException { + if (!(scheduler instanceof FairScheduler)) { + LOG.error("User rule configured for wrong scheduler type."); + throw new IOException( + "User rule can only be configured for the FairScheduler"); + } + + FairScheduler fs = (FairScheduler) scheduler; + queueManager = fs.getQueueManager(); + if (parentRule != null && parentRule.getName().equals(getName())) { + throw new IOException( + "Parent rule should not be the same type as the child rule (User)"); + } + return true; + } + + @Override + public ApplicationPlacementContext getPlacementForApp( + ApplicationSubmissionContext asc, String user) throws YarnException { + String queueName; + + String cleanUser = cleanName(user); + if (parentRule != null) { + LOG.debug("parent rule found: {}", parentRule.getName()); + ApplicationPlacementContext parent = + parentRule.getPlacementForApp(asc, user); + if (parent == null || + queueManager.getQueue(parent.getQueue()) instanceof FSLeafQueue) { + return null; + } + LOG.debug("parent rule result: {}", parent.getQueue()); + queueName = parent.getQueue() + DOT + cleanUser; + } else { + queueName = assureRoot(cleanUser); + } + + // If we can create the queue in the rule or the queue exists return it + if (createQueue || configuredQueue(queueName)) { + return new ApplicationPlacementContext(queueName); + } + return null; + } + + /** + * Check if the queue exists and is part of the configuration i.e. not + * a {@link FSQueue#isDynamic()} queue. + * @param queueName name of the queue to check + * @return true if the queue exists and is a "configured" queue + */ + private boolean configuredQueue(String queueName) { + FSQueue queue = queueManager.getQueue(queueName); + return (queue != null && !queue.isDynamic()); + } +} diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/AllocationConfiguration.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/AllocationConfiguration.java index 826d9f523eb..5bdf96d9e04 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/AllocationConfiguration.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/AllocationConfiguration.java @@ -23,7 +23,6 @@ import java.util.Map; import java.util.Set; -import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.security.authorize.AccessControlList; import org.apache.hadoop.yarn.api.records.ReservationACL; import org.apache.hadoop.yarn.api.records.Resource; @@ -107,6 +106,15 @@ private final Set nonPreemptableQueues; + /** + * Create a fully initialised configuration for the scheduler. + * @param queueProperties The list of queues and their properties from the + * configuration. + * @param allocationFileParser The allocation file parser + * @param newPlacementPolicy An initialised queue placement policy. + * @param globalReservationQueueConfig The reservation queue config + * @throws AllocationConfigurationException + */ public AllocationConfiguration(QueueProperties queueProperties, AllocationFileParser allocationFileParser, QueuePlacementPolicy newPlacementPolicy, @@ -145,7 +153,13 @@ public AllocationConfiguration(QueueProperties queueProperties, queueProperties.getMaxContainerAllocation(); } - public AllocationConfiguration(Configuration conf) { + /** + * Create a base scheduler configuration with just the defaults set. + * Should only be called to init a basic setup on scheduler init. + * @param scheduler The {@link FairScheduler} to create and initialise the + * placement policy. + */ + public AllocationConfiguration(FairScheduler scheduler) { minQueueResources = new HashMap<>(); maxChildQueueResources = new HashMap<>(); maxQueueResources = new HashMap<>(); @@ -170,7 +184,7 @@ public AllocationConfiguration(Configuration conf) { configuredQueues.put(queueType, new HashSet<>()); } placementPolicy = - QueuePlacementPolicy.fromConfiguration(conf, configuredQueues); + QueuePlacementPolicy.fromConfiguration(scheduler); nonPreemptableQueues = new HashSet<>(); queueMaxContainerAllocationMap = new HashMap<>(); } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/AllocationFileLoaderService.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/AllocationFileLoaderService.java index 3300948ce70..a0c70503869 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/AllocationFileLoaderService.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/AllocationFileLoaderService.java @@ -78,6 +78,7 @@ "(?i)(hdfs)|(file)|(s3a)|(viewfs)"; private final Clock clock; + private final FairScheduler scheduler; // Last time we successfully reloaded queues private volatile long lastSuccessfulReload; @@ -95,14 +96,15 @@ private Thread reloadThread; private volatile boolean running = true; - public AllocationFileLoaderService() { - this(SystemClock.getInstance()); + public AllocationFileLoaderService(FairScheduler scheduler) { + this(SystemClock.getInstance(), scheduler); } private List defaultPermissions; - public AllocationFileLoaderService(Clock clock) { + public AllocationFileLoaderService(Clock clock, FairScheduler scheduler) { super(AllocationFileLoaderService.class.getName()); + this.scheduler = scheduler; this.clock = clock; } @@ -255,9 +257,8 @@ public synchronized void reloadAllocations() QueueProperties queueProperties = queueParser.parse(); // Load placement policy and pass it configured queues - Configuration conf = getConfig(); QueuePlacementPolicy newPlacementPolicy = - getQueuePlacementPolicy(allocationFileParser, queueProperties, conf); + getQueuePlacementPolicy(allocationFileParser); setupRootQueueProperties(allocationFileParser, queueProperties); ReservationQueueConfiguration globalReservationQueueConfig = @@ -273,16 +274,14 @@ public synchronized void reloadAllocations() } private QueuePlacementPolicy getQueuePlacementPolicy( - AllocationFileParser allocationFileParser, - QueueProperties queueProperties, Configuration conf) + AllocationFileParser allocationFileParser) throws AllocationConfigurationException { if (allocationFileParser.getQueuePlacementPolicy().isPresent()) { return QueuePlacementPolicy.fromXml( allocationFileParser.getQueuePlacementPolicy().get(), - queueProperties.getConfiguredQueues(), conf); + scheduler); } else { - return QueuePlacementPolicy.fromConfiguration(conf, - queueProperties.getConfiguredQueues()); + return QueuePlacementPolicy.fromConfiguration(scheduler); } } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FairScheduler.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FairScheduler.java index e5d2a066c4a..97354403dc4 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FairScheduler.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FairScheduler.java @@ -57,6 +57,7 @@ import org.apache.hadoop.yarn.server.api.protocolrecords.NMContainerStatus; import org.apache.hadoop.yarn.server.resourcemanager.RMContext; import org.apache.hadoop.yarn.server.resourcemanager.RMCriticalThreadUncaughtExceptionHandler; +import org.apache.hadoop.yarn.server.resourcemanager.placement.ApplicationPlacementContext; import org.apache.hadoop.yarn.server.resourcemanager.recovery.RMStateStore.RMState; import org.apache.hadoop.yarn.server.resourcemanager.reservation.ReservationConstants; import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMApp; @@ -211,7 +212,7 @@ public FairScheduler() { super(FairScheduler.class.getName()); context = new FSContext(this); - allocsLoader = new AllocationFileLoaderService(); + allocsLoader = new AllocationFileLoaderService(this); queueMgr = new QueueManager(this); maxRunningEnforcer = new MaxRunningAppsEnforcer(this); } @@ -220,6 +221,10 @@ public FSContext getContext() { return context; } + public RMContext getRMContext() { + return rmContext; + } + public boolean isAtLeastReservationThreshold( ResourceCalculator resourceCalculator, Resource resource) { return Resources.greaterThanOrEqual(resourceCalculator, @@ -452,7 +457,21 @@ public int getContinuousSchedulingSleepMs() { * configured limits, but the app will not be marked as runnable. */ protected void addApplication(ApplicationId applicationId, - String queueName, String user, boolean isAppRecovering) { + String queueName, String user, boolean isAppRecovering, + ApplicationPlacementContext placementContext) { + // If the placement was rejected the placementContext will be null. + // We ignore placement rules on recovery. + if (!isAppRecovering && placementContext == null) { + String message = + "Reject application " + applicationId + " submitted by user " + user + + " application rejected by placement rules."; + rejectApplicationWithMessage(applicationId, message); + return; + } + // If we get here the queue placement has been run and the queueName + // reflects that already. If we are recovering the application the queue + // was not replaced by the placement rules and the queueName needs to be + // sanity checked if (queueName == null || queueName.isEmpty()) { String message = "Reject application " + applicationId + " submitted by user " + user @@ -470,14 +489,50 @@ protected void addApplication(ApplicationId applicationId, return; } + if (placementContext != null && + !queueName.equalsIgnoreCase(placementContext.getQueue())) { + String message = "Reject application " + applicationId + + " submitted by user " + user + " submit queue (" + queueName + + ") differs from placement queue (" + placementContext.getQueue() + + ")"; + rejectApplicationWithMessage(applicationId, message); + return; + } + writeLock.lock(); try { - RMApp rmApp = rmContext.getRMApps().get(applicationId); - FSLeafQueue queue = assignToQueue(rmApp, queueName, user, applicationId); + // Assign the app to the queue creating and prevent queue delete. + FSLeafQueue queue = queueMgr.getLeafQueue(queueName, true, + applicationId); if (queue == null) { + rejectApplicationWithMessage(applicationId, + queueName + " is not a leaf queue"); return; } + // Enforce ACLs: 2nd check, there could be a time laps between the app + // creation in the RMAppManager and getting here. That means we could + // have a configuration change (prevent race condition) + UserGroupInformation userUgi = UserGroupInformation.createRemoteUser( + user); + + if (!queue.hasAccess(QueueACL.SUBMIT_APPLICATIONS, userUgi) && + !queue.hasAccess(QueueACL.ADMINISTER_QUEUE, userUgi)) { + String msg = "User " + user + " does not have permission to submit " + + applicationId + " to queue " + queueName; + rejectApplicationWithMessage(applicationId, msg); + queue.removeAssignedApp(applicationId); + return; + } + + RMApp rmApp = rmContext.getRMApps().get(applicationId); + if (rmApp != null) { + rmApp.setQueue(queueName); + } else { + LOG.error("Couldn't find RM app for " + applicationId + + " to set queue name on"); + } + if (rmApp != null && rmApp.getAMResourceRequests() != null) { // Resources.fitsIn would always return false when queueMaxShare is 0 // for any resource, but only using Resources.fitsIn is not enough @@ -496,7 +551,7 @@ protected void addApplication(ApplicationId applicationId, + "it has zero amount of resource for a requested " + "resource! Invalid requested AM resources: %s, " + "maximum queue resources: %s", - applicationId, queue.getName(), + applicationId, queueName, invalidAMResourceRequests, queue.getMaxShare()); rejectApplicationWithMessage(applicationId, msg); queue.removeAssignedApp(applicationId); @@ -504,27 +559,13 @@ protected void addApplication(ApplicationId applicationId, } } - // Enforce ACLs - UserGroupInformation userUgi = UserGroupInformation.createRemoteUser( - user); - - if (!queue.hasAccess(QueueACL.SUBMIT_APPLICATIONS, userUgi) && !queue - .hasAccess(QueueACL.ADMINISTER_QUEUE, userUgi)) { - String msg = "User " + userUgi.getUserName() - + " cannot submit applications to queue " + queue.getName() - + "(requested queuename is " + queueName + ")"; - rejectApplicationWithMessage(applicationId, msg); - queue.removeAssignedApp(applicationId); - return; - } - SchedulerApplication application = - new SchedulerApplication(queue, user); + new SchedulerApplication<>(queue, user); applications.put(applicationId, application); queue.getMetrics().submitApp(user); LOG.info("Accepted application " + applicationId + " from user: " + user - + ", in queue: " + queue.getName() + + ", in queue: " + queueName + ", currently num of applications: " + applications.size()); if (isAppRecovering) { if (LOG.isDebugEnabled()) { @@ -597,60 +638,6 @@ protected void addApplicationAttempt( } } - /** - * Helper method for the tests to assign the app to a queue. - */ - @VisibleForTesting - FSLeafQueue assignToQueue(RMApp rmApp, String queueName, String user) { - return assignToQueue(rmApp, queueName, user, null); - } - - /** - * Helper method that attempts to assign the app to a queue. The method is - * responsible to call the appropriate event-handler if the app is rejected. - */ - private FSLeafQueue assignToQueue(RMApp rmApp, String queueName, String user, - ApplicationId applicationId) { - FSLeafQueue queue = null; - String appRejectMsg = null; - - try { - QueuePlacementPolicy placementPolicy = allocConf.getPlacementPolicy(); - queueName = placementPolicy.assignAppToQueue(queueName, user); - if (queueName == null) { - appRejectMsg = "Application rejected by queue placement policy"; - } else { - queue = queueMgr.getLeafQueue(queueName, true, applicationId); - if (queue == null) { - appRejectMsg = queueName + " is not a leaf queue"; - } - } - } catch (IllegalStateException se) { - appRejectMsg = "Unable to match app " + rmApp.getApplicationId() + - " to a queue placement policy, and no valid terminal queue " + - " placement rule is configured. Please contact an administrator " + - " to confirm that the fair scheduler configuration contains a " + - " valid terminal queue placement rule."; - } catch (InvalidQueueNameException qne) { - appRejectMsg = qne.getMessage(); - } catch (IOException ioe) { - // IOException should only happen for a user without groups - appRejectMsg = "Error assigning app to a queue: " + ioe.getMessage(); - } - - if (appRejectMsg != null && rmApp != null) { - rejectApplicationWithMessage(rmApp.getApplicationId(), appRejectMsg); - return null; - } - - if (rmApp != null) { - rmApp.setQueue(queue.getName()); - } else { - LOG.error("Couldn't find RM app to set queue name on"); - } - return queue; - } - private void removeApplication(ApplicationId applicationId, RMAppState finalState) { SchedulerApplication application = applications.remove( @@ -1268,7 +1255,8 @@ public void handle(SchedulerEvent event) { if (queueName != null) { addApplication(appAddedEvent.getApplicationId(), queueName, appAddedEvent.getUser(), - appAddedEvent.getIsAppRecovering()); + appAddedEvent.getIsAppRecovering(), + appAddedEvent.getPlacementContext()); } break; case APP_REMOVED: @@ -1445,12 +1433,8 @@ private void initScheduler(Configuration conf) throws IOException { // This stores per-application scheduling information this.applications = new ConcurrentHashMap<>(); - allocConf = new AllocationConfiguration(conf); - try { - queueMgr.initialize(conf); - } catch (Exception e) { - throw new IOException("Failed to start FairScheduler", e); - } + allocConf = new AllocationConfiguration(this); + queueMgr.initialize(); if (continuousSchedulingEnabled) { // Continuous scheduling is deprecated log it on startup diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/QueueManager.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/QueueManager.java index a9c06780069..456bb1319f9 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/QueueManager.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/QueueManager.java @@ -106,8 +106,7 @@ public FSParentQueue getRootQueue() { return rootQueue; } - public void initialize(Configuration conf) throws IOException, - SAXException, AllocationConfigurationException, ParserConfigurationException { + public void initialize() { // Policies of root and default queue are set to // SchedulingPolicy.DEFAULT_POLICY since the allocation file hasn't been // loaded yet. diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/QueuePlacementPolicy.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/QueuePlacementPolicy.java index 30ea213529d..1dbbeecc804 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/QueuePlacementPolicy.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/QueuePlacementPolicy.java @@ -23,85 +23,161 @@ import java.util.HashMap; import java.util.List; import java.util.Map; -import java.util.Set; +import com.google.common.annotations.VisibleForTesting; import org.apache.hadoop.classification.InterfaceAudience.Private; import org.apache.hadoop.classification.InterfaceStability.Unstable; import org.apache.hadoop.conf.Configuration; -import org.apache.hadoop.security.Groups; -import org.apache.hadoop.util.ReflectionUtils; +import org.apache.hadoop.yarn.server.resourcemanager.placement.DefaultPlacementRule; +import org.apache.hadoop.yarn.server.resourcemanager.placement.PlacementRule; +import org.apache.hadoop.yarn.server.resourcemanager.placement.PrimaryGroupPlacementRule; +import org.apache.hadoop.yarn.server.resourcemanager.placement.RejectPlacementRule; +import org.apache.hadoop.yarn.server.resourcemanager.placement.SecondaryGroupExistingPlacementRule; +import org.apache.hadoop.yarn.server.resourcemanager.placement.SpecifiedPlacementRule; +import org.apache.hadoop.yarn.server.resourcemanager.placement.UserPlacementRule; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; import org.w3c.dom.Element; import org.w3c.dom.Node; import org.w3c.dom.NodeList; +import static org.apache.hadoop.yarn.server.resourcemanager.placement.PlacementFactory.getPlacementRule; + @Private @Unstable public class QueuePlacementPolicy { - private static final Map> ruleClasses; + private static final Logger LOG = + LoggerFactory.getLogger(QueuePlacementPolicy.class); + + private static final Map> ruleClasses; static { - Map> map = - new HashMap>(); - map.put("user", QueuePlacementRule.User.class); - map.put("primaryGroup", QueuePlacementRule.PrimaryGroup.class); + Map> map = + new HashMap<>(); + map.put("user", UserPlacementRule.class); + map.put("primaryGroup", PrimaryGroupPlacementRule.class); map.put("secondaryGroupExistingQueue", - QueuePlacementRule.SecondaryGroupExistingQueue.class); - map.put("specified", QueuePlacementRule.Specified.class); - map.put("nestedUserQueue", - QueuePlacementRule.NestedUserQueue.class); - map.put("default", QueuePlacementRule.Default.class); - map.put("reject", QueuePlacementRule.Reject.class); + SecondaryGroupExistingPlacementRule.class); + map.put("specified", SpecifiedPlacementRule.class); + map.put("nestedUserQueue", UserPlacementRule.class); + map.put("default", DefaultPlacementRule.class); + map.put("reject", RejectPlacementRule.class); ruleClasses = Collections.unmodifiableMap(map); } - - private final List rules; - private final Map> configuredQueues; - private final Groups groups; - - public QueuePlacementPolicy(List rules, - Map> configuredQueues, Configuration conf) + + private List rules = null; + + public QueuePlacementPolicy(List newRules, + FairScheduler fs) throws AllocationConfigurationException { - for (int i = 0; i < rules.size()-1; i++) { - if (rules.get(i).isTerminal()) { + LOG.debug("Placement rule order check"); + for (int i = 0; i < newRules.size()-1; i++) { + if (newRules.get(i).isTerminal()) { throw new AllocationConfigurationException("Rules after rule " - + i + " in queue placement policy can never be reached"); + + (i+1) + " in queue placement policy can never be reached"); } } - if (!rules.get(rules.size()-1).isTerminal()) { + if (!newRules.get(newRules.size()-1).isTerminal()) { throw new AllocationConfigurationException( "Could get past last queue placement rule without assigning"); } - this.rules = rules; - this.configuredQueues = configuredQueues; - groups = new Groups(conf); + // Set the scheduler in the rule to get queues etc + LOG.debug("Initialising new rule set"); + try { + for (PlacementRule rule: newRules){ + rule.initialize(fs); + } + } catch (IOException ioe) { + // We should never throw as we pass in a FS object, however we still + // should consider any exception here a config error. + LOG.debug("Initialising rule set failed", ioe); + throw new AllocationConfigurationException( + "Rule initialisation failed with exception", ioe); + } + // Update the placement manager with the new rule list. + // We only get here when all rules are OK. + rules = newRules; + fs.getRMContext().getQueuePlacementManager().updateRules(newRules); + LOG.debug("PlacementManager active with new rule set"); + } + + /** + * Convenience method for testing only + * @return list of {@link PlacementRule} that are configured + */ + @VisibleForTesting + public List getRules() { + return Collections.unmodifiableList(rules); } - + /** * Builds a QueuePlacementPolicy from an xml element. + * @param confElement the placement policy xml snippet from the + * {@link FairSchedulerConfiguration} + * @param fs the reference to the scheduler needed in the rule on init. + * @return instance of this policy */ - public static QueuePlacementPolicy fromXml(Element el, - Map> configuredQueues, Configuration conf) + public static QueuePlacementPolicy fromXml(Element confElement, + FairScheduler fs) throws AllocationConfigurationException { - List rules = new ArrayList(); - NodeList elements = el.getChildNodes(); + LOG.debug("Reloading placement policy from allocation config"); + if (confElement == null || !confElement.hasChildNodes()) { + throw new AllocationConfigurationException( + "Empty configuration for QueuePlacementPolicy is not allowed"); + } + List newRules = new ArrayList<>(); + NodeList elements = confElement.getChildNodes(); for (int i = 0; i < elements.getLength(); i++) { Node node = elements.item(i); if (node instanceof Element) { - QueuePlacementRule rule = createAndInitializeRule(node); - rules.add(rule); + if (LOG.isDebugEnabled()) { + LOG.debug("Creating new rule: {}", ((Element) node).getAttribute("name")); + } + PlacementRule rule = createRule((Element)node); + + // the only child node that we know is a parent rule + NodeList childList = ((Element)node).getChildNodes(); + for (int j = 0; j < childList.getLength(); j++) { + Node child = childList.item(j); + if (child instanceof Element) { + Element childEl = (Element) child; + if ("rule".equals(childEl.getTagName())) { + String name = childEl.getAttribute("name"); + LOG.debug("Creating new parent rule: {}", name); + if (name.equals("reject") || name.equals("nestedUserQueue")) { + throw new AllocationConfigurationException( + "Rule '" + name + "' is not allowed as a parent rule (" + + ((Element) node).getAttribute("name") + ")"); + } + PlacementRule parentRule = createRule(childEl); + rule.setParentRule(parentRule); + // Init the rule, we do not want to add it to the list of the + // placement manager + try { + parentRule.initialize(fs); + } catch (IOException ioe) { + // We should never throw as we pass in a FS object, however we + // still should consider any exception here a config error. + throw new AllocationConfigurationException( + "Parent Rule initialisation failed with exception", ioe); + } + break; + } + } + } + newRules.add(rule); } } - return new QueuePlacementPolicy(rules, configuredQueues, conf); + return new QueuePlacementPolicy(newRules, fs); } - + /** - * Create and initialize a rule given a xml node - * @param node - * @return QueuePlacementPolicy + * Create a rule from a given a xml node + * @param element the xml element to create the rule from + * @return PlacementRule * @throws AllocationConfigurationException */ - public static QueuePlacementRule createAndInitializeRule(Node node) + private static PlacementRule createRule(Element element) throws AllocationConfigurationException { - Element element = (Element) node; String ruleName = element.getAttribute("name"); if ("".equals(ruleName)) { @@ -109,72 +185,47 @@ public static QueuePlacementRule createAndInitializeRule(Node node) + "rule element"); } - Class clazz = ruleClasses.get(ruleName); + Class clazz = ruleClasses.get(ruleName); if (clazz == null) { throw new AllocationConfigurationException("No rule class found for " + ruleName); } - QueuePlacementRule rule = ReflectionUtils.newInstance(clazz, null); - rule.initializeFromXml(element); - return rule; + return getPlacementRule(clazz, Element.class, element); } /** - * Build a simple queue placement policy from the allow-undeclared-pools and - * user-as-default-queue configuration options. + * Build a simple queue placement policy from the configuration options + * {@link FairSchedulerConfiguration#ALLOW_UNDECLARED_POOLS} and + * {@link FairSchedulerConfiguration#USER_AS_DEFAULT_QUEUE}. + * @param fs the reference to the scheduler needed in the rule on init. + * @return instance of this policy */ - public static QueuePlacementPolicy fromConfiguration(Configuration conf, - Map> configuredQueues) { + public static QueuePlacementPolicy fromConfiguration(FairScheduler fs) { + LOG.debug("Creating base placement policy from config"); + Configuration conf = fs.getConfig(); + boolean create = conf.getBoolean( FairSchedulerConfiguration.ALLOW_UNDECLARED_POOLS, FairSchedulerConfiguration.DEFAULT_ALLOW_UNDECLARED_POOLS); boolean userAsDefaultQueue = conf.getBoolean( FairSchedulerConfiguration.USER_AS_DEFAULT_QUEUE, FairSchedulerConfiguration.DEFAULT_USER_AS_DEFAULT_QUEUE); - List rules = new ArrayList(); - rules.add(new QueuePlacementRule.Specified().initialize(create, null)); + List newRules = new ArrayList<>(); + Class clazz = ruleClasses.get("specified"); + newRules.add(getPlacementRule(clazz, boolean.class, create)); if (userAsDefaultQueue) { - rules.add(new QueuePlacementRule.User().initialize(create, null)); + clazz = ruleClasses.get("user"); + newRules.add(getPlacementRule(clazz, boolean.class, create)); } if (!userAsDefaultQueue || !create) { - rules.add(new QueuePlacementRule.Default().initialize(true, null)); + clazz = ruleClasses.get("default"); + newRules.add(getPlacementRule(clazz, boolean.class, true)); } try { - return new QueuePlacementPolicy(rules, configuredQueues, conf); + return new QueuePlacementPolicy(newRules, fs); } catch (AllocationConfigurationException ex) { throw new RuntimeException("Should never hit exception when loading" + "placement policy from conf", ex); } } - - /** - * Applies this rule to an app with the given requested queue and user/group - * information. - * - * @param requestedQueue - * The queue specified in the ApplicationSubmissionContext - * @param user - * The user submitting the app - * @return - * The name of the queue to assign the app to. Or null if the app should - * be rejected. - * @throws IOException - * If an exception is encountered while getting the user's groups - */ - public String assignAppToQueue(String requestedQueue, String user) - throws IOException { - for (QueuePlacementRule rule : rules) { - String queue = rule.assignAppToQueue(requestedQueue, user, groups, - configuredQueues); - if (queue == null || !queue.isEmpty()) { - return queue; - } - } - throw new IllegalStateException("Should have applied a rule before " + - "reaching here"); - } - - public List getRules() { - return rules; - } } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/QueuePlacementRule.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/QueuePlacementRule.java deleted file mode 100644 index 2c4add4e258..00000000000 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/QueuePlacementRule.java +++ /dev/null @@ -1,366 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair; - -import java.io.IOException; -import java.util.HashMap; -import java.util.List; -import java.util.Map; -import java.util.Set; - -import org.apache.commons.logging.Log; -import org.apache.commons.logging.LogFactory; -import org.apache.hadoop.classification.InterfaceAudience.Private; -import org.apache.hadoop.classification.InterfaceStability.Unstable; -import org.apache.hadoop.security.Groups; -import org.apache.hadoop.yarn.conf.YarnConfiguration; -import org.w3c.dom.Element; -import org.w3c.dom.NamedNodeMap; -import org.w3c.dom.Node; -import org.w3c.dom.NodeList; - -import com.google.common.annotations.VisibleForTesting; - -@Private -@Unstable -public abstract class QueuePlacementRule { - protected boolean create; - public static final Log LOG = - LogFactory.getLog(QueuePlacementRule.class.getName()); - - /** - * Initializes the rule with any arguments. - * - * @param args - * Additional attributes of the rule's xml element other than create. - */ - public QueuePlacementRule initialize(boolean create, Map args) { - this.create = create; - return this; - } - - /** - * - * @param requestedQueue - * The queue explicitly requested. - * @param user - * The user submitting the app. - * @param groups - * The groups of the user submitting the app. - * @param configuredQueues - * The queues specified in the scheduler configuration. - * @return - * The queue to place the app into. An empty string indicates that we should - * continue to the next rule, and null indicates that the app should be rejected. - */ - public String assignAppToQueue(String requestedQueue, String user, - Groups groups, Map> configuredQueues) - throws IOException { - String queue = getQueueForApp(requestedQueue, user, groups, - configuredQueues); - if (create || configuredQueues.get(FSQueueType.LEAF).contains(queue) - || configuredQueues.get(FSQueueType.PARENT).contains(queue)) { - return queue; - } else { - return ""; - } - } - - public void initializeFromXml(Element el) - throws AllocationConfigurationException { - boolean create = true; - NamedNodeMap attributes = el.getAttributes(); - Map args = new HashMap(); - for (int i = 0; i < attributes.getLength(); i++) { - Node node = attributes.item(i); - String key = node.getNodeName(); - String value = node.getNodeValue(); - if (key.equals("create")) { - create = Boolean.parseBoolean(value); - } else { - args.put(key, value); - } - } - initialize(create, args); - } - - /** - * Returns true if this rule never tells the policy to continue. - */ - public abstract boolean isTerminal(); - - /** - * Applies this rule to an app with the given requested queue and user/group - * information. - * - * @param requestedQueue - * The queue specified in the ApplicationSubmissionContext - * @param user - * The user submitting the app. - * @param groups - * The groups of the user submitting the app. - * @return - * The name of the queue to assign the app to, or null to empty string - * continue to the next rule. - */ - protected abstract String getQueueForApp(String requestedQueue, String user, - Groups groups, Map> configuredQueues) - throws IOException; - - /** - * Places apps in queues by username of the submitter - */ - public static class User extends QueuePlacementRule { - @Override - protected String getQueueForApp(String requestedQueue, String user, - Groups groups, Map> configuredQueues) { - return "root." + cleanName(user); - } - - @Override - public boolean isTerminal() { - return create; - } - } - - /** - * Places apps in queues by primary group of the submitter - */ - public static class PrimaryGroup extends QueuePlacementRule { - @Override - protected String getQueueForApp(String requestedQueue, String user, - Groups groups, Map> configuredQueues) - throws IOException { - final List groupList = groups.getGroups(user); - if (groupList.isEmpty()) { - throw new IOException("No groups returned for user " + user); - } - return "root." + cleanName(groupList.get(0)); - } - - @Override - public boolean isTerminal() { - return create; - } - } - - /** - * Places apps in queues by secondary group of the submitter - * - * Match will be made on first secondary group that exist in - * queues - */ - public static class SecondaryGroupExistingQueue extends QueuePlacementRule { - @Override - protected String getQueueForApp(String requestedQueue, String user, - Groups groups, Map> configuredQueues) - throws IOException { - List groupNames = groups.getGroups(user); - for (int i = 1; i < groupNames.size(); i++) { - String group = cleanName(groupNames.get(i)); - if (configuredQueues.get(FSQueueType.LEAF).contains("root." + group) - || configuredQueues.get(FSQueueType.PARENT).contains( - "root." + group)) { - return "root." + group; - } - } - - return ""; - } - - @Override - public boolean isTerminal() { - return false; - } - } - - /** - * Places apps in queues with name of the submitter under the queue - * returned by the nested rule. - */ - public static class NestedUserQueue extends QueuePlacementRule { - @VisibleForTesting - QueuePlacementRule nestedRule; - - /** - * Parse xml and instantiate the nested rule - */ - @Override - public void initializeFromXml(Element el) - throws AllocationConfigurationException { - NodeList elements = el.getChildNodes(); - - for (int i = 0; i < elements.getLength(); i++) { - Node node = elements.item(i); - if (node instanceof Element) { - Element element = (Element) node; - if ("rule".equals(element.getTagName())) { - QueuePlacementRule rule = QueuePlacementPolicy - .createAndInitializeRule(node); - if (rule == null) { - throw new AllocationConfigurationException( - "Unable to create nested rule in nestedUserQueue rule"); - } - this.nestedRule = rule; - break; - } else { - continue; - } - } - } - - if (this.nestedRule == null) { - throw new AllocationConfigurationException( - "No nested rule specified in rule"); - } - super.initializeFromXml(el); - } - - @Override - protected String getQueueForApp(String requestedQueue, String user, - Groups groups, Map> configuredQueues) - throws IOException { - // Apply the nested rule - String queueName = nestedRule.assignAppToQueue(requestedQueue, user, - groups, configuredQueues); - - if (queueName != null && queueName.length() != 0) { - if (!queueName.startsWith("root.")) { - queueName = "root." + queueName; - } - - // Verify if the queue returned by the nested rule is an configured leaf queue, - // if yes then skip to next rule in the queue placement policy - if (configuredQueues.get(FSQueueType.LEAF).contains(queueName)) { - return ""; - } - return queueName + "." + cleanName(user); - } - return queueName; - } - - @Override - public boolean isTerminal() { - return false; - } - } - - /** - * Places apps in queues by requested queue of the submitter - */ - public static class Specified extends QueuePlacementRule { - @Override - protected String getQueueForApp(String requestedQueue, String user, - Groups groups, Map> configuredQueues) { - if (requestedQueue.equals(YarnConfiguration.DEFAULT_QUEUE_NAME)) { - return ""; - } else { - if (!requestedQueue.startsWith("root.")) { - requestedQueue = "root." + requestedQueue; - } - return requestedQueue; - } - } - - @Override - public boolean isTerminal() { - return false; - } - } - - /** - * Places apps in the specified default queue. If no default queue is - * specified the app is placed in root.default queue. - */ - public static class Default extends QueuePlacementRule { - @VisibleForTesting - String defaultQueueName; - - @Override - public QueuePlacementRule initialize(boolean create, - Map args) { - if (defaultQueueName == null) { - defaultQueueName = "root." + YarnConfiguration.DEFAULT_QUEUE_NAME; - } - return super.initialize(create, args); - } - - @Override - public void initializeFromXml(Element el) - throws AllocationConfigurationException { - defaultQueueName = el.getAttribute("queue"); - if (defaultQueueName != null && !defaultQueueName.isEmpty()) { - if (!defaultQueueName.startsWith("root.")) { - defaultQueueName = "root." + defaultQueueName; - } - } else { - defaultQueueName = "root." + YarnConfiguration.DEFAULT_QUEUE_NAME; - } - super.initializeFromXml(el); - } - - @Override - protected String getQueueForApp(String requestedQueue, String user, - Groups groups, Map> configuredQueues) { - return defaultQueueName; - } - - @Override - public boolean isTerminal() { - return true; - } - } - - /** - * Rejects all apps - */ - public static class Reject extends QueuePlacementRule { - @Override - public String assignAppToQueue(String requestedQueue, String user, - Groups groups, Map> configuredQueues) { - return null; - } - - @Override - protected String getQueueForApp(String requestedQueue, String user, - Groups groups, Map> configuredQueues) { - throw new UnsupportedOperationException(); - } - - @Override - public boolean isTerminal() { - return true; - } - } - - /** - * Replace the periods in the username or groupname with "_dot_" and - * remove trailing and leading whitespace. - */ - protected String cleanName(String name) { - name = name.trim(); - if (name.contains(".")) { - String converted = name.replaceAll("\\.", "_dot_"); - LOG.warn("Name " + name + " is converted to " + converted - + " when it is used as a queue name."); - return converted; - } else { - return name; - } - } -} diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestAppManagerWithFairScheduler.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestAppManagerWithFairScheduler.java index feb7ed246ce..b8148c1f0f2 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestAppManagerWithFairScheduler.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestAppManagerWithFairScheduler.java @@ -18,32 +18,30 @@ package org.apache.hadoop.yarn.server.resourcemanager; +import static junit.framework.TestCase.assertTrue; import static org.apache.hadoop.yarn.exceptions.InvalidResourceRequestException.InvalidResourceType.GREATER_THEN_MAX_ALLOCATION; import static org.mockito.Matchers.any; -import static org.mockito.Matchers.matches; -import static org.mockito.Mockito.doAnswer; import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; import java.io.File; import java.io.FileWriter; import java.io.IOException; import java.io.PrintWriter; -import java.util.HashMap; +import java.util.Collections; +import org.apache.hadoop.security.AccessControlException; import org.apache.hadoop.test.GenericTestUtils; import org.apache.hadoop.yarn.MockApps; -import org.apache.hadoop.yarn.api.records.ApplicationAccessType; import org.apache.hadoop.yarn.api.records.ApplicationId; import org.apache.hadoop.yarn.api.records.ApplicationSubmissionContext; import org.apache.hadoop.yarn.api.records.ContainerLaunchContext; import org.apache.hadoop.yarn.api.records.Priority; -import org.apache.hadoop.yarn.api.records.QueueInfo; import org.apache.hadoop.yarn.api.records.Resource; +import org.apache.hadoop.yarn.api.records.ResourceRequest; import org.apache.hadoop.yarn.conf.YarnConfiguration; import org.apache.hadoop.yarn.exceptions.InvalidResourceRequestException; import org.apache.hadoop.yarn.exceptions.YarnException; -import org.apache.hadoop.yarn.factories.RecordFactory; -import org.apache.hadoop.yarn.factory.providers.RecordFactoryProvider; import org.apache.hadoop.yarn.server.resourcemanager.placement.ApplicationPlacementContext; import org.apache.hadoop.yarn.server.resourcemanager.placement.PlacementManager; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.ResourceScheduler; @@ -51,39 +49,34 @@ import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.FairSchedulerConfiguration; import org.apache.hadoop.yarn.server.resourcemanager.security.ClientToAMTokenSecretManagerInRM; import org.apache.hadoop.yarn.server.security.ApplicationACLsManager; +import org.apache.hadoop.yarn.util.Records; import org.apache.hadoop.yarn.util.resource.Resources; -import org.junit.AfterClass; +import org.junit.After; import org.junit.Assert; -import org.junit.BeforeClass; +import org.junit.Before; import org.junit.Test; -import org.mockito.invocation.InvocationOnMock; -import org.mockito.stubbing.Answer; /** - * Testing applications being retired from RM with fair scheduler. - * + * Testing RMAppManager application submission with fair scheduler. */ -public class TestAppManagerWithFairScheduler extends AppManagerTestBase{ +public class TestAppManagerWithFairScheduler extends AppManagerTestBase { private static final String TEST_FOLDER = "test-queues"; private static YarnConfiguration conf = new YarnConfiguration(); + private PlacementManager placementMgr; + private TestRMAppManager rmAppManager; + private RMContext rmContext; + private static String allocFile = + GenericTestUtils.getTestDir(TEST_FOLDER).getAbsolutePath(); - @BeforeClass - public static void setup() throws IOException { - String allocFile = - GenericTestUtils.getTestDir(TEST_FOLDER).getAbsolutePath(); - - int queueMaxAllocation = 512; - + @Before + public void setup() throws IOException { + // Basic config with one queue (override in test if needed) PrintWriter out = new PrintWriter(new FileWriter(allocFile)); out.println(""); out.println(""); - out.println(" "); - out.println(" " + queueMaxAllocation - + " mb 1 vcores" + ""); - out.println(" "); - out.println(" "); + out.println(" "); out.println(" "); out.println(""); out.close(); @@ -92,86 +85,204 @@ public static void setup() throws IOException { ResourceScheduler.class); conf.set(FairSchedulerConfiguration.ALLOCATION_FILE, allocFile); + placementMgr = mock(PlacementManager.class); + + MockRM mockRM = new MockRM(conf); + rmContext = mockRM.getRMContext(); + rmContext.setQueuePlacementManager(placementMgr); + ApplicationMasterService masterService = new ApplicationMasterService( + rmContext, rmContext.getScheduler()); + + rmAppManager = new TestRMAppManager(rmContext, + new ClientToAMTokenSecretManagerInRM(), rmContext.getScheduler(), + masterService, new ApplicationACLsManager(conf), conf); } - @AfterClass - public static void teardown(){ + @After + public void teardown(){ File allocFile = GenericTestUtils.getTestDir(TEST_FOLDER); allocFile.delete(); } @Test public void testQueueSubmitWithHighQueueContainerSize() - throws YarnException { + throws YarnException, IOException { + int maxAlloc = + YarnConfiguration.DEFAULT_RM_SCHEDULER_MINIMUM_ALLOCATION_MB; - ApplicationId appId = MockApps.newAppID(1); - RecordFactory recordFactory = RecordFactoryProvider.getRecordFactory(null); - - Resource resource = Resources.createResource( - YarnConfiguration.DEFAULT_RM_SCHEDULER_MINIMUM_ALLOCATION_MB); - - ApplicationSubmissionContext asContext = - recordFactory.newRecordInstance(ApplicationSubmissionContext.class); - asContext.setApplicationId(appId); - asContext.setResource(resource); - asContext.setPriority(Priority.newInstance(0)); - asContext.setAMContainerSpec(mockContainerLaunchContext(recordFactory)); - asContext.setQueue("queueA"); - QueueInfo mockDefaultQueueInfo = mock(QueueInfo.class); - - // Setup a PlacementManager returns a new queue - PlacementManager placementMgr = mock(PlacementManager.class); - doAnswer(new Answer() { - - @Override - public ApplicationPlacementContext answer(InvocationOnMock invocation) - throws Throwable { - return new ApplicationPlacementContext("queueA"); - } - - }).when(placementMgr).placeApplication( - any(ApplicationSubmissionContext.class), matches("test1")); - doAnswer(new Answer() { - - @Override - public ApplicationPlacementContext answer(InvocationOnMock invocation) - throws Throwable { - return new ApplicationPlacementContext("queueB"); - } - - }).when(placementMgr).placeApplication( - any(ApplicationSubmissionContext.class), matches("test2")); - - MockRM newMockRM = new MockRM(conf); - RMContext newMockRMContext = newMockRM.getRMContext(); - newMockRMContext.setQueuePlacementManager(placementMgr); - ApplicationMasterService masterService = new ApplicationMasterService( - newMockRMContext, newMockRMContext.getScheduler()); - - TestRMAppManager newAppMonitor = new TestRMAppManager(newMockRMContext, - new ClientToAMTokenSecretManagerInRM(), newMockRMContext.getScheduler(), - masterService, new ApplicationACLsManager(conf), conf); + // scheduler config with a limited queue + PrintWriter out = new PrintWriter(new FileWriter(allocFile)); + out.println(""); + out.println(""); + out.println(" "); + out.println(" "); + out.println(" " + maxAlloc + " mb 1 vcores"); + out.println(" "); + out.println(" "); + out.println(" "); + out.println(" "); + out.println(" "); + out.println(""); + out.close(); + rmContext.getScheduler().reinitialize(conf, rmContext); - // only user test has permission to submit to 'test' queue + ApplicationId appId = MockApps.newAppID(1); + Resource res = Resources.createResource(maxAlloc + 1); + ApplicationSubmissionContext asContext = createAppSubmitCtx(appId, res); + // Submit to limited queue + when(placementMgr.placeApplication(any(), any())) + .thenReturn(new ApplicationPlacementContext("limited")); try { - newAppMonitor.submitApplication(asContext, "test1"); + rmAppManager.submitApplication(asContext, "test"); Assert.fail("Test should fail on too high allocation!"); } catch (InvalidResourceRequestException e) { Assert.assertEquals(GREATER_THEN_MAX_ALLOCATION, e.getInvalidResourceType()); } - // Should not throw exception - newAppMonitor.submitApplication(asContext, "test2"); + // submit same app but now place it in the unlimited queue + when(placementMgr.placeApplication(any(), any())) + .thenReturn(new ApplicationPlacementContext("root.unlimited")); + rmAppManager.submitApplication(asContext, "test"); } - private static ContainerLaunchContext mockContainerLaunchContext( - RecordFactory recordFactory) { - ContainerLaunchContext amContainer = recordFactory.newRecordInstance( - ContainerLaunchContext.class); - amContainer - .setApplicationACLs(new HashMap()); - return amContainer; + @Test + public void testQueueSubmitWithPermissionLimits() + throws YarnException, IOException { + + conf.set(YarnConfiguration.YARN_ACL_ENABLE, "true"); + + PrintWriter out = new PrintWriter(new FileWriter(allocFile)); + out.println(""); + out.println(""); + out.println(" "); + out.println(" "); + out.println(" "); + out.println(" "); + out.println(" "); + out.println(" "); + out.println(" test "); + out.println(" "); + out.println(" "); + out.println(" "); + out.println(" "); + out.println(" test "); + out.println(" "); + out.println(" "); + out.println(""); + out.close(); + rmContext.getScheduler().reinitialize(conf, rmContext); + + ApplicationId appId = MockApps.newAppID(1); + Resource res = Resources.createResource(1024,1); + ApplicationSubmissionContext asContext = createAppSubmitCtx(appId, res); + + // Submit to no access queue + when(placementMgr.placeApplication(any(), any())) + .thenReturn(new ApplicationPlacementContext("noaccess")); + try { + rmAppManager.submitApplication(asContext, "test"); + Assert.fail("Test should have failed with access denied"); + } catch (YarnException e) { + assertTrue("Access exception not found" + , e.getCause() instanceof AccessControlException); + } + // Submit to submit access queue + when(placementMgr.placeApplication(any(), any())) + .thenReturn(new ApplicationPlacementContext("submitonly")); + rmAppManager.submitApplication(asContext, "test"); + // Submit second app to admin access queue + appId = MockApps.newAppID(2); + asContext = createAppSubmitCtx(appId, res); + when(placementMgr.placeApplication(any(), any())) + .thenReturn(new ApplicationPlacementContext("adminonly")); + rmAppManager.submitApplication(asContext, "test"); + } + + @Test + public void testQueueSubmitWithRootPermission() + throws YarnException, IOException { + + conf.set(YarnConfiguration.YARN_ACL_ENABLE, "true"); + + PrintWriter out = new PrintWriter(new FileWriter(allocFile)); + out.println(""); + out.println(""); + out.println(" "); + out.println(" "); + out.println(" "); + out.println(" "); + out.println(" "); + out.println(" "); + out.println(""); + out.close(); + rmContext.getScheduler().reinitialize(conf, rmContext); + + ApplicationId appId = MockApps.newAppID(1); + Resource res = Resources.createResource(1024,1); + ApplicationSubmissionContext asContext = createAppSubmitCtx(appId, res); + + // Submit to noaccess queue should be allowed by root ACL + when(placementMgr.placeApplication(any(), any())) + .thenReturn(new ApplicationPlacementContext("noaccess")); + rmAppManager.submitApplication(asContext, "test"); + } + + @Test + public void testQueueSubmitWithAutoCreateQueue() + throws YarnException, IOException { + + conf.set(YarnConfiguration.YARN_ACL_ENABLE, "true"); + + PrintWriter out = new PrintWriter(new FileWriter(allocFile)); + out.println(""); + out.println(""); + out.println(" "); + out.println(" "); + out.println(" "); + out.println(" "); + out.println(" "); + out.println(" "); + out.println(" test "); + out.println(" "); + out.println(" "); + out.println(""); + out.close(); + rmContext.getScheduler().reinitialize(conf, rmContext); + + ApplicationId appId = MockApps.newAppID(1); + Resource res = Resources.createResource(1024,1); + ApplicationSubmissionContext asContext = createAppSubmitCtx(appId, res); + + // Submit to noaccess parent with non existent child queue + when(placementMgr.placeApplication(any(), any())) + .thenReturn(new ApplicationPlacementContext("root.noaccess.child")); + try { + rmAppManager.submitApplication(asContext, "test"); + Assert.fail("Test should have failed with access denied"); + } catch (YarnException e) { + assertTrue("Access exception not found" + , e.getCause() instanceof AccessControlException); + } + // Submit to submitonly parent with non existent child queue + when(placementMgr.placeApplication(any(), any())) + .thenReturn(new ApplicationPlacementContext("root.submitonly.child")); + rmAppManager.submitApplication(asContext, "test"); + } + + private ApplicationSubmissionContext createAppSubmitCtx(ApplicationId appId, + Resource res) { + ApplicationSubmissionContext asContext = + Records.newRecord(ApplicationSubmissionContext.class); + asContext.setApplicationId(appId); + ResourceRequest resReg = + ResourceRequest.newInstance(Priority.newInstance(0), + ResourceRequest.ANY, res, 1); + asContext.setAMContainerResourceRequests( + Collections.singletonList(resReg)); + asContext.setAMContainerSpec(mock(ContainerLaunchContext.class)); + asContext.setQueue("default"); + return asContext; } } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/TestSchedulerUtils.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/TestSchedulerUtils.java index 696924203de..a6eb0ee9879 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/TestSchedulerUtils.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/TestSchedulerUtils.java @@ -84,6 +84,7 @@ import org.apache.hadoop.yarn.server.resourcemanager.TestAMAuthorization.MyContainerManager; import org.apache.hadoop.yarn.server.resourcemanager.nodelabels.NullRMNodeLabelsManager; import org.apache.hadoop.yarn.server.resourcemanager.nodelabels.RMNodeLabelsManager; +import org.apache.hadoop.yarn.server.resourcemanager.placement.ApplicationPlacementContext; import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMApp; import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMAppState; import org.apache.hadoop.yarn.server.resourcemanager.rmapp.attempt.RMAppAttempt; @@ -1016,10 +1017,12 @@ public static void waitSchedulerApplicationAttemptStopped( Map> applications, EventHandler handler, String queueName) { + ApplicationPlacementContext apc = + new ApplicationPlacementContext(queueName); ApplicationId appId = ApplicationId.newInstance(System.currentTimeMillis(), 1); AppAddedSchedulerEvent appAddedEvent = - new AppAddedSchedulerEvent(appId, queueName, "user"); + new AppAddedSchedulerEvent(appId, queueName, "user", apc); handler.handle(appAddedEvent); SchedulerApplication app = applications.get(appId); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FairSchedulerTestBase.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FairSchedulerTestBase.java index 4f1f20b942b..e67d1aa152f 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FairSchedulerTestBase.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FairSchedulerTestBase.java @@ -35,6 +35,7 @@ import org.apache.hadoop.yarn.server.resourcemanager.ResourceManager; import org.apache.hadoop.yarn.server.resourcemanager.MockNodes; import org.apache.hadoop.yarn.server.resourcemanager.nodelabels.RMNodeLabelsManager; +import org.apache.hadoop.yarn.server.resourcemanager.placement.ApplicationPlacementContext; import org.apache.hadoop.yarn.server.resourcemanager.rmnode.RMNode; import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMApp; import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMAppEvent; @@ -177,7 +178,11 @@ protected ApplicationAttemptId createSchedulingRequest( Collection requests, String queueId, String userId) { ApplicationAttemptId id = createAppAttemptId(this.APP_ID++, this.ATTEMPT_ID++); - scheduler.addApplication(id.getApplicationId(), queueId, userId, false); + // This fakes the placement which is not part of the scheduler anymore + ApplicationPlacementContext placementCtx = + new ApplicationPlacementContext(queueId); + scheduler.addApplication(id.getApplicationId(), queueId, userId, false, + placementCtx); // This conditional is for testAclSubmitApplication where app is rejected // and no app is added. if (scheduler.getSchedulerApplications() @@ -212,7 +217,11 @@ protected ApplicationAttemptId createSchedulingRequest(String queueId, String userId, List ask) { ApplicationAttemptId id = createAppAttemptId(this.APP_ID++, this.ATTEMPT_ID++); - scheduler.addApplication(id.getApplicationId(), queueId, userId, false); + // This fakes the placement which is not part of the scheduler anymore + ApplicationPlacementContext placementCtx = + new ApplicationPlacementContext(queueId); + scheduler.addApplication(id.getApplicationId(), queueId, userId, false, + placementCtx); // This conditional is for testAclSubmitApplication where app is rejected // and no app is added. if (scheduler.getSchedulerApplications().containsKey(id.getApplicationId())) { @@ -298,8 +307,11 @@ private void addApplication(String queue, String user, ApplicationId appId) { resourceManager.getRMContext().getRMApps().get(appId).handle(event); event = new RMAppEvent(appId, RMAppEventType.APP_ACCEPTED); resourceManager.getRMContext().getRMApps().get(appId).handle(event); + // This fakes the placement which is not part of the scheduler anymore + ApplicationPlacementContext placementCtx = + new ApplicationPlacementContext(queue); AppAddedSchedulerEvent appAddedEvent = new AppAddedSchedulerEvent( - appId, queue, user); + appId, queue, user, placementCtx); scheduler.handle(appAddedEvent); } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestAllocationFileLoaderService.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestAllocationFileLoaderService.java index ac30b237472..3bca964a1b7 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestAllocationFileLoaderService.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestAllocationFileLoaderService.java @@ -25,15 +25,22 @@ import org.apache.hadoop.hdfs.MiniDFSCluster; import org.apache.hadoop.yarn.api.records.Resource; import org.apache.hadoop.yarn.conf.YarnConfiguration; +import org.apache.hadoop.yarn.server.resourcemanager.RMContext; +import org.apache.hadoop.yarn.server.resourcemanager.placement.DefaultPlacementRule; +import org.apache.hadoop.yarn.server.resourcemanager.placement.PlacementManager; +import org.apache.hadoop.yarn.server.resourcemanager.placement.PlacementRule; +import org.apache.hadoop.yarn.server.resourcemanager.placement.PrimaryGroupPlacementRule; +import org.apache.hadoop.yarn.server.resourcemanager.placement.SpecifiedPlacementRule; +import org.apache.hadoop.yarn.server.resourcemanager.placement.UserPlacementRule; import org.apache.hadoop.yarn.server.resourcemanager.reservation.ReservationSchedulerConfiguration; -import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.QueuePlacementRule.NestedUserQueue; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.allocationfile.AllocationFileWriter; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.policies.DominantResourceFairnessPolicy; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.policies.FairSharePolicy; import org.apache.hadoop.yarn.util.ControlledClock; -import org.apache.hadoop.yarn.util.resource.ResourceUtils; +import org.apache.hadoop.yarn.util.SystemClock; import org.apache.hadoop.yarn.util.resource.Resources; import org.apache.hadoop.yarn.util.resource.TestResourceUtils; +import org.junit.Before; import org.junit.Test; import java.io.File; import java.io.FileOutputStream; @@ -44,7 +51,6 @@ import java.net.URISyntaxException; import java.net.URL; import java.nio.charset.StandardCharsets; -import java.util.HashMap; import java.util.List; import java.util.Map; import java.util.Set; @@ -54,6 +60,8 @@ import static org.junit.Assert.assertNull; import static org.junit.Assert.assertTrue; import static org.junit.Assert.fail; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; public class TestAllocationFileLoaderService { @@ -66,10 +74,28 @@ "test-queues").getAbsolutePath(); private static final String TEST_FAIRSCHED_XML = "test-fair-scheduler.xml"; + private FairScheduler scheduler; + private Configuration conf; + + @Before + public void setup() { + SystemClock clock = SystemClock.getInstance(); + PlacementManager placementManager = new PlacementManager(); + FairSchedulerConfiguration fsConf = new FairSchedulerConfiguration(); + RMContext rmContext = mock(RMContext.class); + when(rmContext.getQueuePlacementManager()).thenReturn(placementManager); + + scheduler = mock(FairScheduler.class); + conf = new YarnConfiguration(); + when(scheduler.getClock()).thenReturn(clock); + when(scheduler.getConf()).thenReturn(fsConf); + when(scheduler.getConfig()).thenReturn(conf); + when(scheduler.getRMContext()).thenReturn(rmContext); + } + @Test public void testGetAllocationFileFromFileSystem() throws IOException, URISyntaxException { - Configuration conf = new YarnConfiguration(); File baseDir = new File(TEST_DIR + Path.SEPARATOR + "getAllocHDFS").getAbsoluteFile(); FileUtil.fullyDelete(baseDir); @@ -85,7 +111,8 @@ public void testGetAllocationFileFromFileSystem() fs.copyFromLocalFile(new Path(fschedURL.toURI()), new Path(fsAllocPath)); conf.set(FairSchedulerConfiguration.ALLOCATION_FILE, fsAllocPath); - AllocationFileLoaderService allocLoader = new AllocationFileLoaderService(); + AllocationFileLoaderService allocLoader = + new AllocationFileLoaderService(scheduler); Path allocationFile = allocLoader.getAllocationFile(conf); assertEquals(fsAllocPath, allocationFile.toString()); assertTrue(fs.exists(allocationFile)); @@ -96,9 +123,9 @@ public void testGetAllocationFileFromFileSystem() @Test (expected = UnsupportedFileSystemException.class) public void testDenyGetAllocationFileFromUnsupportedFileSystem() throws UnsupportedFileSystemException { - Configuration conf = new YarnConfiguration(); conf.set(FairSchedulerConfiguration.ALLOCATION_FILE, "badfs:///badfile"); - AllocationFileLoaderService allocLoader = new AllocationFileLoaderService(); + AllocationFileLoaderService allocLoader = + new AllocationFileLoaderService(scheduler); allocLoader.getAllocationFile(conf); } @@ -106,12 +133,11 @@ public void testDenyGetAllocationFileFromUnsupportedFileSystem() @Test public void testGetAllocationFileFromClasspath() { try { - Configuration conf = new Configuration(); FileSystem fs = FileSystem.get(conf); conf.set(FairSchedulerConfiguration.ALLOCATION_FILE, TEST_FAIRSCHED_XML); AllocationFileLoaderService allocLoader = - new AllocationFileLoaderService(); + new AllocationFileLoaderService(scheduler); Path allocationFile = allocLoader.getAllocationFile(conf); assertEquals(TEST_FAIRSCHED_XML, allocationFile.getName()); assertTrue(fs.exists(allocationFile)); @@ -137,11 +163,10 @@ public void testReload() throws Exception { ControlledClock clock = new ControlledClock(); clock.setTime(0); - Configuration conf = new Configuration(); conf.set(FairSchedulerConfiguration.ALLOCATION_FILE, ALLOC_FILE); AllocationFileLoaderService allocLoader = new AllocationFileLoaderService( - clock); + clock, scheduler); allocLoader.reloadIntervalMs = 5; allocLoader.init(conf); ReloadListener confHolder = new ReloadListener(); @@ -151,9 +176,9 @@ public void testReload() throws Exception { // Verify conf QueuePlacementPolicy policy = allocConf.getPlacementPolicy(); - List rules = policy.getRules(); + List rules = policy.getRules(); assertEquals(1, rules.size()); - assertEquals(QueuePlacementRule.Default.class, rules.get(0).getClass()); + assertEquals(DefaultPlacementRule.class, rules.get(0).getClass()); assertEquals(1, allocConf.getQueueMaxApps("root.queueA")); assertEquals(2, allocConf.getConfiguredQueues().get(FSQueueType.LEAF) .size()); @@ -176,7 +201,7 @@ public void testReload() throws Exception { out.println(" "); out.println(" "); out.println(" "); - out.println(" "); + //out.println(" "); out.println(" "); out.println(""); out.close(); @@ -193,13 +218,13 @@ public void testReload() throws Exception { allocConf = confHolder.allocConf; policy = allocConf.getPlacementPolicy(); rules = policy.getRules(); - assertEquals(3, rules.size()); - assertEquals(QueuePlacementRule.Specified.class, rules.get(0).getClass()); - assertEquals(QueuePlacementRule.NestedUserQueue.class, rules.get(1) + assertEquals(2, rules.size()); + assertEquals(SpecifiedPlacementRule.class, rules.get(0).getClass()); + assertEquals(UserPlacementRule.class, rules.get(1) .getClass()); - assertEquals(QueuePlacementRule.PrimaryGroup.class, - ((NestedUserQueue) (rules.get(1))).nestedRule.getClass()); - assertEquals(QueuePlacementRule.Default.class, rules.get(2).getClass()); + assertEquals(PrimaryGroupPlacementRule.class, + (rules.get(1)).getParentRule().getClass()); + //assertEquals(DefaultPlacementRule.class, rules.get(2).getClass()); assertEquals(3, allocConf.getQueueMaxApps("root.queueB")); assertEquals(1, allocConf.getConfiguredQueues().get(FSQueueType.LEAF) .size()); @@ -212,7 +237,8 @@ public void testAllocationFileParsing() throws Exception { Configuration conf = new YarnConfiguration(); TestResourceUtils.addNewTypesToResources(A_CUSTOM_RESOURCE); conf.set(FairSchedulerConfiguration.ALLOCATION_FILE, ALLOC_FILE); - AllocationFileLoaderService allocLoader = new AllocationFileLoaderService(); + AllocationFileLoaderService allocLoader = + new AllocationFileLoaderService(scheduler); AllocationFileWriter .create() @@ -456,9 +482,9 @@ public void testAllocationFileParsing() throws Exception { @Test public void testBackwardsCompatibleAllocationFileParsing() throws Exception { - Configuration conf = new Configuration(); conf.set(FairSchedulerConfiguration.ALLOCATION_FILE, ALLOC_FILE); - AllocationFileLoaderService allocLoader = new AllocationFileLoaderService(); + AllocationFileLoaderService allocLoader = + new AllocationFileLoaderService(scheduler); PrintWriter out = new PrintWriter(new FileWriter(ALLOC_FILE)); out.println(""); @@ -570,7 +596,6 @@ public void testBackwardsCompatibleAllocationFileParsing() throws Exception { @Test public void testSimplePlacementPolicyFromConf() throws Exception { - Configuration conf = new Configuration(); conf.set(FairSchedulerConfiguration.ALLOCATION_FILE, ALLOC_FILE); conf.setBoolean(FairSchedulerConfiguration.ALLOW_UNDECLARED_POOLS, false); conf.setBoolean(FairSchedulerConfiguration.USER_AS_DEFAULT_QUEUE, false); @@ -581,19 +606,25 @@ public void testSimplePlacementPolicyFromConf() throws Exception { out.println(""); out.close(); - AllocationFileLoaderService allocLoader = new AllocationFileLoaderService(); + AllocationFileLoaderService allocLoader = + new AllocationFileLoaderService(scheduler); allocLoader.init(conf); ReloadListener confHolder = new ReloadListener(); allocLoader.setReloadListener(confHolder); allocLoader.reloadAllocations(); AllocationConfiguration allocConf = confHolder.allocConf; + when(scheduler.getAllocationConfiguration()).thenReturn(allocConf); + QueueManager queueManager = new QueueManager(scheduler); + FSQueueMetrics.forQueue("root", null, true, conf); + queueManager.initialize(); + QueuePlacementPolicy placementPolicy = allocConf.getPlacementPolicy(); - List rules = placementPolicy.getRules(); + List rules = placementPolicy.getRules(); assertEquals(2, rules.size()); - assertEquals(QueuePlacementRule.Specified.class, rules.get(0).getClass()); - assertEquals(false, rules.get(0).create); - assertEquals(QueuePlacementRule.Default.class, rules.get(1).getClass()); + assertEquals(SpecifiedPlacementRule.class, rules.get(0).getClass()); + assertEquals(false, rules.get(0).createQueue); + assertEquals(DefaultPlacementRule.class, rules.get(1).getClass()); } /** @@ -602,7 +633,6 @@ public void testSimplePlacementPolicyFromConf() throws Exception { */ @Test (expected = AllocationConfigurationException.class) public void testQueueAlongsideRoot() throws Exception { - Configuration conf = new Configuration(); conf.set(FairSchedulerConfiguration.ALLOCATION_FILE, ALLOC_FILE); PrintWriter out = new PrintWriter(new FileWriter(ALLOC_FILE)); @@ -615,7 +645,8 @@ public void testQueueAlongsideRoot() throws Exception { out.println(""); out.close(); - AllocationFileLoaderService allocLoader = new AllocationFileLoaderService(); + AllocationFileLoaderService allocLoader = + new AllocationFileLoaderService(scheduler); allocLoader.init(conf); ReloadListener confHolder = new ReloadListener(); allocLoader.setReloadListener(confHolder); @@ -628,7 +659,6 @@ public void testQueueAlongsideRoot() throws Exception { */ @Test (expected = AllocationConfigurationException.class) public void testQueueNameContainingPeriods() throws Exception { - Configuration conf = new Configuration(); conf.set(FairSchedulerConfiguration.ALLOCATION_FILE, ALLOC_FILE); PrintWriter out = new PrintWriter(new FileWriter(ALLOC_FILE)); @@ -639,7 +669,8 @@ public void testQueueNameContainingPeriods() throws Exception { out.println(""); out.close(); - AllocationFileLoaderService allocLoader = new AllocationFileLoaderService(); + AllocationFileLoaderService allocLoader = + new AllocationFileLoaderService(scheduler); allocLoader.init(conf); ReloadListener confHolder = new ReloadListener(); allocLoader.setReloadListener(confHolder); @@ -652,7 +683,6 @@ public void testQueueNameContainingPeriods() throws Exception { */ @Test (expected = AllocationConfigurationException.class) public void testQueueNameContainingOnlyWhitespace() throws Exception { - Configuration conf = new Configuration(); conf.set(FairSchedulerConfiguration.ALLOCATION_FILE, ALLOC_FILE); PrintWriter out = new PrintWriter(new FileWriter(ALLOC_FILE)); @@ -663,7 +693,8 @@ public void testQueueNameContainingOnlyWhitespace() throws Exception { out.println(""); out.close(); - AllocationFileLoaderService allocLoader = new AllocationFileLoaderService(); + AllocationFileLoaderService allocLoader = + new AllocationFileLoaderService(scheduler); allocLoader.init(conf); ReloadListener confHolder = new ReloadListener(); allocLoader.setReloadListener(confHolder); @@ -672,7 +703,6 @@ public void testQueueNameContainingOnlyWhitespace() throws Exception { @Test public void testParentTagWithReservation() throws Exception { - Configuration conf = new Configuration(); conf.set(FairSchedulerConfiguration.ALLOCATION_FILE, ALLOC_FILE); PrintWriter out = new PrintWriter(new FileWriter(ALLOC_FILE)); @@ -685,7 +715,8 @@ public void testParentTagWithReservation() throws Exception { out.println(""); out.close(); - AllocationFileLoaderService allocLoader = new AllocationFileLoaderService(); + AllocationFileLoaderService allocLoader = + new AllocationFileLoaderService(scheduler); allocLoader.init(conf); ReloadListener confHolder = new ReloadListener(); allocLoader.setReloadListener(confHolder); @@ -701,7 +732,6 @@ public void testParentTagWithReservation() throws Exception { @Test public void testParentWithReservation() throws Exception { - Configuration conf = new Configuration(); conf.set(FairSchedulerConfiguration.ALLOCATION_FILE, ALLOC_FILE); PrintWriter out = new PrintWriter(new FileWriter(ALLOC_FILE)); @@ -716,7 +746,8 @@ public void testParentWithReservation() throws Exception { out.println(""); out.close(); - AllocationFileLoaderService allocLoader = new AllocationFileLoaderService(); + AllocationFileLoaderService allocLoader = + new AllocationFileLoaderService(scheduler); allocLoader.init(conf); ReloadListener confHolder = new ReloadListener(); allocLoader.setReloadListener(confHolder); @@ -732,7 +763,6 @@ public void testParentWithReservation() throws Exception { @Test public void testParentTagWithChild() throws Exception { - Configuration conf = new Configuration(); conf.set(FairSchedulerConfiguration.ALLOCATION_FILE, ALLOC_FILE); PrintWriter out = new PrintWriter(new FileWriter(ALLOC_FILE)); @@ -745,7 +775,8 @@ public void testParentTagWithChild() throws Exception { out.println(""); out.close(); - AllocationFileLoaderService allocLoader = new AllocationFileLoaderService(); + AllocationFileLoaderService allocLoader = + new AllocationFileLoaderService(scheduler); allocLoader.init(conf); ReloadListener confHolder = new ReloadListener(); allocLoader.setReloadListener(confHolder); @@ -764,7 +795,6 @@ public void testParentTagWithChild() throws Exception { */ @Test (expected = AllocationConfigurationException.class) public void testQueueNameContainingNBWhitespace() throws Exception { - Configuration conf = new Configuration(); conf.set(FairSchedulerConfiguration.ALLOCATION_FILE, ALLOC_FILE); PrintWriter out = new PrintWriter(new OutputStreamWriter( @@ -776,7 +806,8 @@ public void testQueueNameContainingNBWhitespace() throws Exception { out.println(""); out.close(); - AllocationFileLoaderService allocLoader = new AllocationFileLoaderService(); + AllocationFileLoaderService allocLoader = + new AllocationFileLoaderService(scheduler); allocLoader.init(conf); ReloadListener confHolder = new ReloadListener(); allocLoader.setReloadListener(confHolder); @@ -788,7 +819,6 @@ public void testQueueNameContainingNBWhitespace() throws Exception { */ @Test (expected = AllocationConfigurationException.class) public void testDefaultQueueSchedulingModeIsFIFO() throws Exception { - Configuration conf = new Configuration(); conf.set(FairSchedulerConfiguration.ALLOCATION_FILE, ALLOC_FILE); PrintWriter out = new PrintWriter(new FileWriter(ALLOC_FILE)); @@ -798,7 +828,8 @@ public void testDefaultQueueSchedulingModeIsFIFO() throws Exception { out.println(""); out.close(); - AllocationFileLoaderService allocLoader = new AllocationFileLoaderService(); + AllocationFileLoaderService allocLoader = + new AllocationFileLoaderService(scheduler); allocLoader.init(conf); ReloadListener confHolder = new ReloadListener(); allocLoader.setReloadListener(confHolder); @@ -807,7 +838,6 @@ public void testDefaultQueueSchedulingModeIsFIFO() throws Exception { @Test public void testReservableQueue() throws Exception { - Configuration conf = new Configuration(); conf.set(FairSchedulerConfiguration.ALLOCATION_FILE, ALLOC_FILE); PrintWriter out = new PrintWriter(new FileWriter(ALLOC_FILE)); @@ -824,7 +854,8 @@ public void testReservableQueue() throws Exception { out.println(""); out.close(); - AllocationFileLoaderService allocLoader = new AllocationFileLoaderService(); + AllocationFileLoaderService allocLoader = + new AllocationFileLoaderService(scheduler); allocLoader.init(conf); ReloadListener confHolder = new ReloadListener(); allocLoader.setReloadListener(confHolder); @@ -871,7 +902,6 @@ public void testReservableQueue() throws Exception { @Test (expected = AllocationConfigurationException.class) public void testReservableCannotBeCombinedWithDynamicUserQueue() throws Exception { - Configuration conf = new Configuration(); conf.set(FairSchedulerConfiguration.ALLOCATION_FILE, ALLOC_FILE); PrintWriter out = new PrintWriter(new FileWriter(ALLOC_FILE)); @@ -884,7 +914,8 @@ public void testReservableCannotBeCombinedWithDynamicUserQueue() out.println(""); out.close(); - AllocationFileLoaderService allocLoader = new AllocationFileLoaderService(); + AllocationFileLoaderService allocLoader = + new AllocationFileLoaderService(scheduler); allocLoader.init(conf); ReloadListener confHolder = new ReloadListener(); allocLoader.setReloadListener(confHolder); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestAppRunnability.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestAppRunnability.java index f5819357ba0..003a947ac07 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestAppRunnability.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestAppRunnability.java @@ -34,6 +34,7 @@ import org.apache.hadoop.yarn.api.records.Resource; import org.apache.hadoop.yarn.server.resourcemanager.MockNodes; import org.apache.hadoop.yarn.server.resourcemanager.MockRM; +import org.apache.hadoop.yarn.server.resourcemanager.placement.ApplicationPlacementContext; import org.apache.hadoop.yarn.server.resourcemanager.rmapp.attempt.RMAppAttemptState; import org.apache.hadoop.yarn.server.resourcemanager.rmnode.RMNode; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.QueueMetrics; @@ -78,7 +79,7 @@ public void testUserAsDefaultQueue() throws Exception { conf.set(FairSchedulerConfiguration.USER_AS_DEFAULT_QUEUE, "true"); scheduler.reinitialize(conf, resourceManager.getRMContext()); ApplicationAttemptId appAttemptId = createAppAttemptId(1, 1); - createApplicationWithAMResource(appAttemptId, "default", "user1", null); + createApplicationWithAMResource(appAttemptId, "root.user1", "user1", null); assertEquals(1, scheduler.getQueueManager().getLeafQueue("user1", true) .getNumRunnableApps()); assertEquals(0, scheduler.getQueueManager().getLeafQueue("default", true) @@ -110,9 +111,11 @@ public void testNotUserAsDefaultQueue() throws Exception { @Test public void testAppAdditionAndRemoval() throws Exception { ApplicationAttemptId attemptId = createAppAttemptId(1, 1); + ApplicationPlacementContext apc = + new ApplicationPlacementContext("user1"); AppAddedSchedulerEvent appAddedEvent = - new AppAddedSchedulerEvent(attemptId.getApplicationId(), "default", - "user1"); + new AppAddedSchedulerEvent(attemptId.getApplicationId(), "user1", + "user1", apc); scheduler.handle(appAddedEvent); AppAttemptAddedSchedulerEvent attemptAddedEvent = new AppAttemptAddedSchedulerEvent(createAppAttemptId(1, 1), false); @@ -149,7 +152,7 @@ public void testPreemptionVariablesForQueueCreatedRuntime() throws Exception { // User1 submits one application ApplicationAttemptId appAttemptId = createAppAttemptId(1, 1); - createApplicationWithAMResource(appAttemptId, "default", "user1", null); + createApplicationWithAMResource(appAttemptId, "user1", "user1", null); // The user1 queue should inherit the configurations from the root queue FSLeafQueue userQueue = @@ -184,12 +187,14 @@ public void testDontAllowUndeclaredPools() throws Exception { FSLeafQueue jerryQueue = queueManager.getLeafQueue("jerry", false); FSLeafQueue defaultQueue = queueManager.getLeafQueue("default", false); + // NOTE: placement is not inside the scheduler anymore need to fake it here. + // The scheduling request contains the fake placing // Should get put into jerry createSchedulingRequest(1024, "jerry", "someuser"); assertEquals(1, jerryQueue.getNumRunnableApps()); // Should get forced into default - createSchedulingRequest(1024, "newqueue", "someuser"); + createSchedulingRequest(1024, "default", "someuser"); assertEquals(1, jerryQueue.getNumRunnableApps()); assertEquals(1, defaultQueue.getNumRunnableApps()); @@ -200,7 +205,7 @@ public void testDontAllowUndeclaredPools() throws Exception { assertEquals(2, defaultQueue.getNumRunnableApps()); // Should get put into jerry because of user-as-default-queue - createSchedulingRequest(1024, "default", "jerry"); + createSchedulingRequest(1024, "jerry", "jerry"); assertEquals(2, jerryQueue.getNumRunnableApps()); assertEquals(2, defaultQueue.getNumRunnableApps()); } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestContinuousScheduling.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestContinuousScheduling.java index e6a841a9c20..c2a0ad1e936 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestContinuousScheduling.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestContinuousScheduling.java @@ -29,6 +29,7 @@ import org.apache.hadoop.yarn.exceptions.YarnRuntimeException; import org.apache.hadoop.yarn.server.resourcemanager.MockNodes; import org.apache.hadoop.yarn.server.resourcemanager.MockRM; +import org.apache.hadoop.yarn.server.resourcemanager.placement.ApplicationPlacementContext; import org.apache.hadoop.yarn.server.resourcemanager.rmcontainer.RMContainer; import org.apache.hadoop.yarn.server.resourcemanager.rmnode.RMNode; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.ClusterNodeTracker; @@ -120,7 +121,10 @@ public void testBasic() throws InterruptedException { createAppAttemptId(this.APP_ID++, this.ATTEMPT_ID++); createMockRMApp(appAttemptId); - scheduler.addApplication(appAttemptId.getApplicationId(), "queue11", "user11", false); + ApplicationPlacementContext placementCtx = + new ApplicationPlacementContext("queue11"); + scheduler.addApplication(appAttemptId.getApplicationId(), "queue11", + "user11", false, placementCtx); scheduler.addApplicationAttempt(appAttemptId, false, false); List ask = new ArrayList<>(); ask.add(createResourceRequest(1024, 1, ResourceRequest.ANY, 1, 1, true)); @@ -156,8 +160,10 @@ public void testSortedNodes() throws Exception { createAppAttemptId(this.APP_ID++, this.ATTEMPT_ID++); createMockRMApp(appAttemptId); - scheduler.addApplication(appAttemptId.getApplicationId(), - "queue11", "user11", false); + ApplicationPlacementContext placementCtx = + new ApplicationPlacementContext("queue11"); + scheduler.addApplication(appAttemptId.getApplicationId(), "queue11", + "user11", false, placementCtx); scheduler.addApplicationAttempt(appAttemptId, false, false); List ask = new ArrayList<>(); ResourceRequest request = @@ -353,8 +359,10 @@ public void testFairSchedulerContinuousSchedulingInitTime() throws Exception { id11 = createAppAttemptId(1, 1); createMockRMApp(id11); priority = Priority.newInstance(priorityValue); + ApplicationPlacementContext placementCtx = + new ApplicationPlacementContext("root.queue1"); scheduler.addApplication(id11.getApplicationId(), "root.queue1", "user1", - false); + false, placementCtx); scheduler.addApplicationAttempt(id11, false, false); fsAppAttempt = scheduler.getApplicationAttempt(id11); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFSAppAttempt.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFSAppAttempt.java index 51ffd23fd2c..3719f748d7f 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFSAppAttempt.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFSAppAttempt.java @@ -35,6 +35,7 @@ import org.apache.hadoop.yarn.api.records.ApplicationAttemptId; import org.apache.hadoop.yarn.api.records.Priority; import org.apache.hadoop.yarn.server.resourcemanager.RMContext; +import org.apache.hadoop.yarn.server.resourcemanager.placement.ApplicationPlacementContext; import org.apache.hadoop.yarn.server.resourcemanager.rmnode.RMNode; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.NodeType; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.QueueMetrics; @@ -298,8 +299,10 @@ public void testHeadroomWithBlackListedNodes() { assertEquals(0, clusterUsage.getVirtualCores()); ApplicationAttemptId id11 = createAppAttemptId(1, 1); createMockRMApp(id11); + ApplicationPlacementContext placementCtx = + new ApplicationPlacementContext("default"); scheduler.addApplication(id11.getApplicationId(), - "default", "user1", false); + "default", "user1", false, placementCtx); scheduler.addApplicationAttempt(id11, false, false); assertNotNull(scheduler.getSchedulerApplications().get(id11. getApplicationId())); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFSParentQueue.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFSParentQueue.java index c6ff5aaa33d..32da3a7c5fa 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFSParentQueue.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFSParentQueue.java @@ -18,6 +18,8 @@ package org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair; +import org.apache.hadoop.yarn.server.resourcemanager.RMContext; +import org.apache.hadoop.yarn.server.resourcemanager.placement.PlacementManager; import org.apache.hadoop.yarn.util.SystemClock; import org.apache.hadoop.yarn.util.resource.DefaultResourceCalculator; import org.junit.Before; @@ -29,23 +31,27 @@ public class TestFSParentQueue { - private FairSchedulerConfiguration conf; private QueueManager queueManager; @Before - public void setUp() throws Exception { - conf = new FairSchedulerConfiguration(); + public void setUp() { + FairSchedulerConfiguration conf = new FairSchedulerConfiguration(); + RMContext rmContext = mock(RMContext.class); + SystemClock clock = SystemClock.getInstance(); + PlacementManager placementManager = new PlacementManager(); FairScheduler scheduler = mock(FairScheduler.class); - AllocationConfiguration allocConf = new AllocationConfiguration(conf); - when(scheduler.getAllocationConfiguration()).thenReturn(allocConf); + when(scheduler.getRMContext()).thenReturn(rmContext); + when(scheduler.getConfig()).thenReturn(conf); when(scheduler.getConf()).thenReturn(conf); when(scheduler.getResourceCalculator()).thenReturn( new DefaultResourceCalculator()); - SystemClock clock = SystemClock.getInstance(); when(scheduler.getClock()).thenReturn(clock); + when(rmContext.getQueuePlacementManager()).thenReturn(placementManager); + AllocationConfiguration allocConf = new AllocationConfiguration(scheduler); + when(scheduler.getAllocationConfiguration()).thenReturn(allocConf); queueManager = new QueueManager(scheduler); FSQueueMetrics.forQueue("root", null, true, conf); - queueManager.initialize(conf); + queueManager.initialize(); } @Test diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFairScheduler.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFairScheduler.java index 0d6caebac68..c6ab436cab0 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFairScheduler.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFairScheduler.java @@ -37,10 +37,7 @@ import java.util.Arrays; import java.util.Collection; import java.util.Collections; -import java.util.HashMap; -import java.util.HashSet; import java.util.List; -import java.util.Map; import java.util.Set; import java.util.stream.Collectors; @@ -86,6 +83,9 @@ import org.apache.hadoop.yarn.server.resourcemanager.NodeManager; import org.apache.hadoop.yarn.server.resourcemanager.RMContext; import org.apache.hadoop.yarn.server.resourcemanager.RMContextImpl; +import org.apache.hadoop.yarn.server.resourcemanager.placement.ApplicationPlacementContext; +import org.apache.hadoop.yarn.server.resourcemanager.placement.DefaultPlacementRule; +import org.apache.hadoop.yarn.server.resourcemanager.placement.PlacementRule; import org.apache.hadoop.yarn.server.resourcemanager.recovery.MemoryRMStateStore; import org.apache.hadoop.yarn.server.resourcemanager.rmapp.MockRMApp; import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMApp; @@ -115,7 +115,6 @@ import org.apache.hadoop.yarn.server.resourcemanager.scheduler.event.NodeAddedSchedulerEvent; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.event.NodeRemovedSchedulerEvent; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.event.NodeUpdateSchedulerEvent; -import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.QueuePlacementRule.Default; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.policies.DominantResourceFairnessPolicy; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.policies.FifoPolicy; import org.apache.hadoop.yarn.server.utils.BuilderUtils; @@ -431,8 +430,11 @@ public void testNormalizationUsingQueueMaximumAllocation() private void allocateAppAttempt(String queueName, int id, int memorySize) { ApplicationAttemptId id11 = createAppAttemptId(id, id); createMockRMApp(id11); + ApplicationPlacementContext placementCtx = + new ApplicationPlacementContext(queueName); + scheduler.addApplication(id11.getApplicationId(), queueName, "user1", - false); + false, placementCtx); scheduler.addApplicationAttempt(id11, false, false); List ask1 = new ArrayList(); ResourceRequest request1 = @@ -1384,8 +1386,10 @@ public void testRackLocalAppReservationThreshold() throws Exception { createAppAttemptId(this.APP_ID++, this.ATTEMPT_ID++); createMockRMApp(attemptId); + ApplicationPlacementContext placementCtx = + new ApplicationPlacementContext("queue1"); scheduler.addApplication(attemptId.getApplicationId(), "queue1", "user1", - false); + false, placementCtx); scheduler.addApplicationAttempt(attemptId, false, false); List asks = new ArrayList(); asks.add(createResourceRequest(2048, node2.getRackName(), 1, 1, false)); @@ -1794,7 +1798,8 @@ public void testEmptyQueueName() throws Exception { // only default queue assertEquals(1, scheduler.getQueueManager().getLeafQueues().size()); - // submit app with empty queue + // Submit app with empty queue + // Submit fails before we reach the placement check. ApplicationAttemptId appAttemptId = createAppAttemptId(1, 1); AppAddedSchedulerEvent appAddedEvent = new AppAddedSchedulerEvent(appAttemptId.getApplicationId(), "", "user1"); @@ -1815,7 +1820,8 @@ public void testQueueuNameWithPeriods() throws Exception { // only default queue assertEquals(1, scheduler.getQueueManager().getLeafQueues().size()); - // submit app with queue name (.A) + // Submit app with queue name (.A) + // Submit fails before we reach the placement check. ApplicationAttemptId appAttemptId1 = createAppAttemptId(1, 1); AppAddedSchedulerEvent appAddedEvent1 = new AppAddedSchedulerEvent(appAttemptId1.getApplicationId(), ".A", "user1"); @@ -1825,7 +1831,8 @@ public void testQueueuNameWithPeriods() throws Exception { assertNull(scheduler.getSchedulerApp(appAttemptId1)); assertEquals(0, resourceManager.getRMContext().getRMApps().size()); - // submit app with queue name (A.) + // Submit app with queue name (A.) + // Submit fails before we reach the placement check. ApplicationAttemptId appAttemptId2 = createAppAttemptId(2, 1); AppAddedSchedulerEvent appAddedEvent2 = new AppAddedSchedulerEvent(appAttemptId2.getApplicationId(), "A.", "user1"); @@ -1836,16 +1843,18 @@ public void testQueueuNameWithPeriods() throws Exception { assertEquals(0, resourceManager.getRMContext().getRMApps().size()); // submit app with queue name (A.B) + // Submit does not fail we must have a placement context. ApplicationAttemptId appAttemptId3 = createAppAttemptId(3, 1); AppAddedSchedulerEvent appAddedEvent3 = - new AppAddedSchedulerEvent(appAttemptId3.getApplicationId(), "A.B", "user1"); + new AppAddedSchedulerEvent(appAttemptId3.getApplicationId(), "A.B", + "user1", new ApplicationPlacementContext("A.B")); scheduler.handle(appAddedEvent3); // submission accepted assertEquals(2, scheduler.getQueueManager().getLeafQueues().size()); assertNull(scheduler.getSchedulerApp(appAttemptId3)); assertEquals(0, resourceManager.getRMContext().getRMApps().size()); } - +/* @Test public void testAssignToQueue() throws Exception { conf.set(FairSchedulerConfiguration.USER_AS_DEFAULT_QUEUE, "true"); @@ -1962,7 +1971,7 @@ public void testQueuePlacementWithPolicy() throws Exception { appId = createSchedulingRequest(1024, "default", "otheruser"); assertEquals("root.default", scheduler.getSchedulerApp(appId).getQueueName()); } - +*/ @Test public void testFairShareWithMinAlloc() throws Exception { conf.set(FairSchedulerConfiguration.ALLOCATION_FILE, ALLOC_FILE); @@ -2007,38 +2016,6 @@ else if (p.getName().equals("root.queueB")) { } } } - - @Test - public void testNestedUserQueue() throws IOException { - conf.set(FairSchedulerConfiguration.ALLOCATION_FILE, ALLOC_FILE); - conf.setClass(CommonConfigurationKeys.HADOOP_SECURITY_GROUP_MAPPING, - SimpleGroupsMapping.class, GroupMappingServiceProvider.class); - PrintWriter out = new PrintWriter(new FileWriter(ALLOC_FILE)); - out.println(""); - out.println(""); - out.println(""); - out.println("1024mb,0vcores"); - out.println(""); - out.println(""); - out.println(""); - out.println(""); - out.println(" "); - out.println(""); - out.println(""); - out.println(""); - out.println(""); - out.close(); - - scheduler.init(conf); - scheduler.start(); - scheduler.reinitialize(conf, resourceManager.getRMContext()); - RMApp rmApp1 = new MockRMApp(0, 0, RMAppState.NEW); - - FSLeafQueue user1Leaf = scheduler.assignToQueue(rmApp1, "root.default", - "user1"); - - assertEquals("root.user1group.user1", user1Leaf.getName()); - } @Test public void testFairShareAndWeightsInNestedUserQueueRule() throws Exception { @@ -2208,7 +2185,7 @@ public void testSteadyFairShareWithQueueCreatedRuntime() throws Exception { // Submit one application ApplicationAttemptId appAttemptId1 = createAppAttemptId(1, 1); - createApplicationWithAMResource(appAttemptId1, "default", "user1", null); + createApplicationWithAMResource(appAttemptId1, "user1", "user1", null); assertEquals(3072, scheduler.getQueueManager() .getLeafQueue("default", false).getSteadyFairShare().getMemorySize()); assertEquals(3072, scheduler.getQueueManager() @@ -2229,8 +2206,10 @@ public void testQueueDemandCalculation() throws Exception { // First ask, queue1 requests 1 large (minReqSize * 2). ApplicationAttemptId id11 = createAppAttemptId(1, 1); createMockRMApp(id11); + ApplicationPlacementContext placementCtx = + new ApplicationPlacementContext("root.queue1"); scheduler.addApplication(id11.getApplicationId(), - "root.queue1", "user1", false); + "root.queue1", "user1", false, placementCtx); scheduler.addApplicationAttempt(id11, false, false); List ask1 = new ArrayList(); ResourceRequest request1 = createResourceRequest(minReqSize * 2, @@ -2242,8 +2221,9 @@ public void testQueueDemandCalculation() throws Exception { // Second ask, queue2 requests 1 large. ApplicationAttemptId id21 = createAppAttemptId(2, 1); createMockRMApp(id21); + placementCtx = new ApplicationPlacementContext("root.queue2"); scheduler.addApplication(id21.getApplicationId(), - "root.queue2", "user1", false); + "root.queue2", "user1", false, placementCtx); scheduler.addApplicationAttempt(id21, false, false); List ask2 = new ArrayList(); ResourceRequest request2 = createResourceRequest(2 * minReqSize, @@ -2259,7 +2239,7 @@ public void testQueueDemandCalculation() throws Exception { ApplicationAttemptId id22 = createAppAttemptId(2, 2); createMockRMApp(id22); scheduler.addApplication(id22.getApplicationId(), - "root.queue2", "user1", false); + "root.queue2", "user1", false, placementCtx); scheduler.addApplicationAttempt(id22, false, false); List ask3 = new ArrayList(); ResourceRequest request4 = createResourceRequest(minReqSize, @@ -2866,8 +2846,10 @@ public void testMultipleNodesSingleRackRequest() throws Exception { createAppAttemptId(this.APP_ID++, this.ATTEMPT_ID++); createMockRMApp(attemptId); + ApplicationPlacementContext placementCtx = + new ApplicationPlacementContext("queue1"); scheduler.addApplication(attemptId.getApplicationId(), "queue1", "user1", - false); + false, placementCtx); scheduler.addApplicationAttempt(attemptId, false, false); // 1 request with 2 nodes on the same rack. another request with 1 node on @@ -3202,7 +3184,10 @@ public void testNotAllowSubmitApplication() throws Exception { ApplicationAttemptId attId = ApplicationAttemptId.newInstance(applicationId, this.ATTEMPT_ID++); - scheduler.addApplication(attId.getApplicationId(), queue, user, false); + ApplicationPlacementContext placementCtx = + new ApplicationPlacementContext(queue); + scheduler.addApplication(attId.getApplicationId(), queue, user, false, + placementCtx); numTries = 0; while (application.getFinishTime() == 0 && numTries < MAX_TRIES) { @@ -4539,8 +4524,10 @@ public void testSchedulingOnRemovedNode() throws Exception { ApplicationAttemptId id11 = createAppAttemptId(1, 1); createMockRMApp(id11); + ApplicationPlacementContext placementCtx = + new ApplicationPlacementContext("root.queue1"); scheduler.addApplication(id11.getApplicationId(), "root.queue1", "user1", - false); + false, placementCtx); scheduler.addApplicationAttempt(id11, false, false); List ask1 = new ArrayList<>(); @@ -4591,12 +4578,11 @@ public void testDefaultRuleInitializesProperlyWhenPolicyNotConfigured() scheduler.start(); scheduler.reinitialize(conf, resourceManager.getRMContext()); - List rules = - scheduler.allocConf.placementPolicy.getRules(); + List rules = scheduler.allocConf.placementPolicy.getRules(); - for (QueuePlacementRule rule : rules) { - if (rule instanceof Default) { - Default defaultRule = (Default) rule; + for (PlacementRule rule : rules) { + if (rule instanceof DefaultPlacementRule) { + DefaultPlacementRule defaultRule = (DefaultPlacementRule) rule; assertNotNull(defaultRule.defaultQueueName); } } @@ -4666,7 +4652,7 @@ public void testGetAppsInQueue() throws Exception { ApplicationAttemptId appAttId2 = createSchedulingRequest(1024, 1, "queue1.subqueue2", "user1"); ApplicationAttemptId appAttId3 = - createSchedulingRequest(1024, 1, "default", "user1"); + createSchedulingRequest(1024, 1, "user1", "user1"); List apps = scheduler.getAppsInQueue("queue1.subqueue1"); @@ -4868,11 +4854,13 @@ public void testDoubleRemoval() throws Exception { scheduler.reinitialize(conf, resourceManager.getRMContext()); ApplicationAttemptId attemptId = createAppAttemptId(1, 1); - // The placement rule will add the app to the user based queue but the + // The placement rule should add it to the user based queue but the // passed in queue must exist. + ApplicationPlacementContext apc = + new ApplicationPlacementContext(testUser); AppAddedSchedulerEvent appAddedEvent = new AppAddedSchedulerEvent(attemptId.getApplicationId(), testUser, - testUser); + testUser, apc); scheduler.handle(appAddedEvent); AppAttemptAddedSchedulerEvent attemptAddedEvent = new AppAttemptAddedSchedulerEvent(createAppAttemptId(1, 1), false); @@ -4916,9 +4904,11 @@ public void testMoveAfterRemoval() throws Exception { scheduler.reinitialize(conf, resourceManager.getRMContext()); ApplicationAttemptId attemptId = createAppAttemptId(1, 1); + ApplicationPlacementContext apc = + new ApplicationPlacementContext(testUser); AppAddedSchedulerEvent appAddedEvent = new AppAddedSchedulerEvent(attemptId.getApplicationId(), testUser, - testUser); + testUser, apc); scheduler.handle(appAddedEvent); AppAttemptAddedSchedulerEvent attemptAddedEvent = new AppAttemptAddedSchedulerEvent(createAppAttemptId(1, 1), false); @@ -4970,8 +4960,10 @@ public void testQueueNameWithTrailingSpace() throws Exception { // submit app with queue name "A" ApplicationAttemptId appAttemptId1 = createAppAttemptId(1, 1); + ApplicationPlacementContext apc = + new ApplicationPlacementContext("A"); AppAddedSchedulerEvent appAddedEvent1 = new AppAddedSchedulerEvent( - appAttemptId1.getApplicationId(), "A", "user1"); + appAttemptId1.getApplicationId(), "A", "user1", apc); scheduler.handle(appAddedEvent1); // submission accepted assertEquals(2, scheduler.getQueueManager().getLeafQueues().size()); @@ -4988,8 +4980,9 @@ public void testQueueNameWithTrailingSpace() throws Exception { // submit app with queue name "A " ApplicationAttemptId appAttemptId2 = createAppAttemptId(2, 1); + apc = new ApplicationPlacementContext("A"); AppAddedSchedulerEvent appAddedEvent2 = new AppAddedSchedulerEvent( - appAttemptId2.getApplicationId(), "A ", "user1"); + appAttemptId2.getApplicationId(), "A ", "user1", apc); scheduler.handle(appAddedEvent2); // submission rejected assertEquals(2, scheduler.getQueueManager().getLeafQueues().size()); @@ -4999,8 +4992,9 @@ public void testQueueNameWithTrailingSpace() throws Exception { // submit app with queue name "B.C" ApplicationAttemptId appAttemptId3 = createAppAttemptId(3, 1); + apc = new ApplicationPlacementContext("B.C"); AppAddedSchedulerEvent appAddedEvent3 = new AppAddedSchedulerEvent( - appAttemptId3.getApplicationId(), "B.C", "user1"); + appAttemptId3.getApplicationId(), "B.C", "user1", apc); scheduler.handle(appAddedEvent3); // submission accepted assertEquals(3, scheduler.getQueueManager().getLeafQueues().size()); @@ -5056,7 +5050,7 @@ public void testUserAsDefaultQueueWithLeadingTrailingSpaceUserName() scheduler.start(); scheduler.reinitialize(conf, resourceManager.getRMContext()); ApplicationAttemptId appAttemptId = createAppAttemptId(1, 1); - createApplicationWithAMResource(appAttemptId, "default", " user1", null); + createApplicationWithAMResource(appAttemptId, "root.user1", " user1", null); assertEquals(1, scheduler.getQueueManager().getLeafQueue("user1", true) .getNumRunnableApps()); assertEquals(0, scheduler.getQueueManager().getLeafQueue("default", true) @@ -5065,7 +5059,7 @@ public void testUserAsDefaultQueueWithLeadingTrailingSpaceUserName() .get(appAttemptId.getApplicationId()).getQueue()); ApplicationAttemptId attId2 = createAppAttemptId(2, 1); - createApplicationWithAMResource(attId2, "default", "user1 ", null); + createApplicationWithAMResource(attId2, "root.user1", "user1 ", null); assertEquals(2, scheduler.getQueueManager().getLeafQueue("user1", true) .getNumRunnableApps()); assertEquals(0, scheduler.getQueueManager().getLeafQueue("default", true) @@ -5074,7 +5068,7 @@ public void testUserAsDefaultQueueWithLeadingTrailingSpaceUserName() .get(attId2.getApplicationId()).getQueue()); ApplicationAttemptId attId3 = createAppAttemptId(3, 1); - createApplicationWithAMResource(attId3, "default", "user1", null); + createApplicationWithAMResource(attId3, "root.user1", "user1", null); assertEquals(3, scheduler.getQueueManager().getLeafQueue("user1", true) .getNumRunnableApps()); assertEquals(0, scheduler.getQueueManager().getLeafQueue("default", true) @@ -5508,8 +5502,10 @@ public void testCompletedContainerOnRemovedNode() throws IOException { // Create application attempt ApplicationAttemptId appAttemptId = createAppAttemptId(1, 1); createMockRMApp(appAttemptId); + ApplicationPlacementContext placementCtx = + new ApplicationPlacementContext("root.queue1"); scheduler.addApplication(appAttemptId.getApplicationId(), "root.queue1", - "user1", false); + "user1", false, placementCtx); scheduler.addApplicationAttempt(appAttemptId, false, false); // Create container request that goes to a specific node. @@ -5594,7 +5590,7 @@ private void testAppRejectedToQueueWithZeroCapacityOfResource(String resource) ResourceRequest amReqs = ResourceRequest.newBuilder() .capability(Resource.newInstance(5 * GB, 3)).build(); - createApplicationWithAMResource(appAttemptId1, "queueA", "user1", + createApplicationWithAMResource(appAttemptId1, "root.queueA", "user1", Resource.newInstance(GB, 1), Lists.newArrayList(amReqs)); scheduler.update(); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFairSchedulerWithMultiResourceTypes.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFairSchedulerWithMultiResourceTypes.java index f9fcf5328f0..aad3b39ab4a 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFairSchedulerWithMultiResourceTypes.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFairSchedulerWithMultiResourceTypes.java @@ -17,22 +17,55 @@ package org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair; import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.ha.HAServiceProtocol; import org.apache.hadoop.yarn.api.protocolrecords.ResourceTypes; +import org.apache.hadoop.yarn.api.records.ApplicationId; +import org.apache.hadoop.yarn.api.records.NodeId; import org.apache.hadoop.yarn.api.records.Resource; import org.apache.hadoop.yarn.api.records.ResourceInformation; +import org.apache.hadoop.yarn.conf.ConfigurationProvider; import org.apache.hadoop.yarn.conf.YarnConfiguration; +import org.apache.hadoop.yarn.event.Dispatcher; +import org.apache.hadoop.yarn.nodelabels.NodeAttributesManager; +import org.apache.hadoop.yarn.server.resourcemanager.*; +import org.apache.hadoop.yarn.server.resourcemanager.ahs.RMApplicationHistoryWriter; +import org.apache.hadoop.yarn.server.resourcemanager.metrics.SystemMetricsPublisher; +import org.apache.hadoop.yarn.server.resourcemanager.nodelabels.RMDelegatedNodeLabelsUpdater; +import org.apache.hadoop.yarn.server.resourcemanager.nodelabels.RMNodeLabelsManager; +import org.apache.hadoop.yarn.server.resourcemanager.placement.PlacementManager; +import org.apache.hadoop.yarn.server.resourcemanager.recovery.RMStateStore; +import org.apache.hadoop.yarn.server.resourcemanager.reservation.ReservationSystem; +import org.apache.hadoop.yarn.server.resourcemanager.resource.ResourceProfilesManager; +import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMApp; +import org.apache.hadoop.yarn.server.resourcemanager.rmapp.attempt.AMLivelinessMonitor; +import org.apache.hadoop.yarn.server.resourcemanager.rmapp.monitor.RMAppLifetimeMonitor; +import org.apache.hadoop.yarn.server.resourcemanager.rmcontainer.ContainerAllocationExpirer; +import org.apache.hadoop.yarn.server.resourcemanager.rmnode.RMNode; +import org.apache.hadoop.yarn.server.resourcemanager.scheduler.ResourceScheduler; +import org.apache.hadoop.yarn.server.resourcemanager.scheduler.SchedulerNode; +import org.apache.hadoop.yarn.server.resourcemanager.scheduler.constraint.AllocationTagsManager; +import org.apache.hadoop.yarn.server.resourcemanager.scheduler.constraint.PlacementConstraintManager; +import org.apache.hadoop.yarn.server.resourcemanager.scheduler.distributed.QueueLimitCalculator; +import org.apache.hadoop.yarn.server.resourcemanager.scheduler.placement.MultiNodeSortingManager; +import org.apache.hadoop.yarn.server.resourcemanager.security.*; +import org.apache.hadoop.yarn.server.resourcemanager.timelineservice.RMTimelineCollectorManager; +import org.apache.hadoop.yarn.server.resourcemanager.volume.csi.VolumeManager; import org.apache.hadoop.yarn.util.resource.ResourceUtils; import org.junit.After; import org.junit.Before; import org.junit.Test; import java.io.IOException; +import java.nio.ByteBuffer; import java.util.HashMap; import java.util.Map; +import java.util.concurrent.ConcurrentMap; import static org.apache.hadoop.yarn.util.resource.ResourceUtils.MAXIMUM_ALLOCATION; import static org.apache.hadoop.yarn.util.resource.ResourceUtils.UNITS; import static org.junit.Assert.assertEquals; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; public class TestFairSchedulerWithMultiResourceTypes extends FairSchedulerTestBase { @@ -44,6 +77,11 @@ public void setUp() throws IOException { scheduler = new FairScheduler(); conf = createConfiguration(); initResourceTypes(conf); + // since this runs outside of the normal context we need to set one + RMContext rmContext = mock(RMContext.class); + PlacementManager placementManager = new PlacementManager(); + when(rmContext.getQueuePlacementManager()).thenReturn(placementManager); + scheduler.setRMContext(rmContext); } @After diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestMaxRunningAppsEnforcer.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestMaxRunningAppsEnforcer.java index 964ecf560d4..c6923495d1c 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestMaxRunningAppsEnforcer.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestMaxRunningAppsEnforcer.java @@ -27,10 +27,10 @@ import java.util.List; import java.util.Map; -import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.yarn.server.resourcemanager.RMContext; import org.apache.hadoop.yarn.api.records.ApplicationAttemptId; import org.apache.hadoop.yarn.api.records.ApplicationId; +import org.apache.hadoop.yarn.server.resourcemanager.placement.PlacementManager; import org.apache.hadoop.yarn.util.ControlledClock; import org.apache.hadoop.yarn.util.resource.DefaultResourceCalculator; import org.junit.Before; @@ -46,26 +46,27 @@ private FairScheduler scheduler; @Before - public void setup() throws Exception { - Configuration conf = new Configuration(); + public void setup() { + FairSchedulerConfiguration conf = new FairSchedulerConfiguration(); + PlacementManager placementManager = new PlacementManager(); + rmContext = mock(RMContext.class); + when(rmContext.getQueuePlacementManager()).thenReturn(placementManager); + when(rmContext.getEpoch()).thenReturn(0L); clock = new ControlledClock(); scheduler = mock(FairScheduler.class); - when(scheduler.getConf()).thenReturn( - new FairSchedulerConfiguration(conf)); + when(scheduler.getConf()).thenReturn(conf); + when(scheduler.getConfig()).thenReturn(conf); when(scheduler.getClock()).thenReturn(clock); - AllocationConfiguration allocConf = new AllocationConfiguration( - conf); - when(scheduler.getAllocationConfiguration()).thenReturn(allocConf); when(scheduler.getResourceCalculator()).thenReturn( new DefaultResourceCalculator()); - + when(scheduler.getRMContext()).thenReturn(rmContext); + AllocationConfiguration allocConf = new AllocationConfiguration(scheduler); + when(scheduler.getAllocationConfiguration()).thenReturn(allocConf); queueManager = new QueueManager(scheduler); - queueManager.initialize(conf); + queueManager.initialize(); userMaxApps = allocConf.userMaxApps; maxAppsEnforcer = new MaxRunningAppsEnforcer(scheduler); appNum = 0; - rmContext = mock(RMContext.class); - when(rmContext.getEpoch()).thenReturn(0L); } private FSAppAttempt addApp(FSLeafQueue queue, String user) { diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestQueueManager.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestQueueManager.java index 9d8c96079f6..07cc2062765 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestQueueManager.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestQueueManager.java @@ -26,6 +26,7 @@ import org.apache.hadoop.yarn.api.records.ApplicationAttemptId; import org.apache.hadoop.yarn.api.records.ApplicationId; import org.apache.hadoop.yarn.server.resourcemanager.RMContext; +import org.apache.hadoop.yarn.server.resourcemanager.placement.PlacementManager; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.ActiveUsersManager; import org.apache.hadoop.yarn.util.SystemClock; import org.apache.hadoop.yarn.util.resource.DefaultResourceCalculator; @@ -38,34 +39,38 @@ import com.google.common.collect.Sets; public class TestQueueManager { - private FairSchedulerConfiguration conf; private QueueManager queueManager; private FairScheduler scheduler; @Before - public void setUp() throws Exception { - conf = new FairSchedulerConfiguration(); + public void setUp() { + PlacementManager placementManager = new PlacementManager(); + FairSchedulerConfiguration conf = new FairSchedulerConfiguration(); + RMContext rmContext = mock(RMContext.class); + when(rmContext.getQueuePlacementManager()).thenReturn(placementManager); + SystemClock clock = SystemClock.getInstance(); + scheduler = mock(FairScheduler.class); + when(scheduler.getConf()).thenReturn(conf); + when(scheduler.getConfig()).thenReturn(conf); + when(scheduler.getRMContext()).thenReturn(rmContext); + when(scheduler.getResourceCalculator()).thenReturn( + new DefaultResourceCalculator()); + when(scheduler.getClock()).thenReturn(clock); - AllocationConfiguration allocConf = new AllocationConfiguration(conf); + AllocationConfiguration allocConf = + new AllocationConfiguration(scheduler); + when(scheduler.getAllocationConfiguration()).thenReturn(allocConf); // Set up some queues to test default child max resource inheritance allocConf.configuredQueues.get(FSQueueType.PARENT).add("root.test"); allocConf.configuredQueues.get(FSQueueType.LEAF).add("root.test.childA"); allocConf.configuredQueues.get(FSQueueType.PARENT).add("root.test.childB"); - when(scheduler.getAllocationConfiguration()).thenReturn(allocConf); - when(scheduler.getConf()).thenReturn(conf); - when(scheduler.getResourceCalculator()).thenReturn( - new DefaultResourceCalculator()); - - SystemClock clock = SystemClock.getInstance(); - - when(scheduler.getClock()).thenReturn(clock); queueManager = new QueueManager(scheduler); FSQueueMetrics.forQueue("root", null, true, conf); - queueManager.initialize(conf); + queueManager.initialize(); queueManager.updateAllocationConfiguration(allocConf); } @@ -118,7 +123,8 @@ public void testReloadTurnsLeafQueueIntoParent() { */ @Test public void testReloadTurnsLeafToParentWithNoLeaf() { - AllocationConfiguration allocConf = new AllocationConfiguration(conf); + AllocationConfiguration allocConf = + new AllocationConfiguration(scheduler); // Create a leaf queue1 allocConf.configuredQueues.get(FSQueueType.LEAF).add("root.queue1"); queueManager.updateAllocationConfiguration(allocConf); @@ -130,7 +136,7 @@ public void testReloadTurnsLeafToParentWithNoLeaf() { FSLeafQueue q1 = queueManager.getLeafQueue("queue1", false); ApplicationId appId = ApplicationId.newInstance(0, 0); q1.addAssignedApp(appId); - allocConf = new AllocationConfiguration(conf); + allocConf = new AllocationConfiguration(scheduler); allocConf.configuredQueues.get(FSQueueType.PARENT) .add("root.queue1"); @@ -169,7 +175,8 @@ public void testCheckQueueNodeName() { private void updateConfiguredLeafQueues(QueueManager queueMgr, String... confLeafQueues) { - AllocationConfiguration allocConf = new AllocationConfiguration(conf); + AllocationConfiguration allocConf = + new AllocationConfiguration(scheduler); allocConf.configuredQueues.get(FSQueueType.LEAF) .addAll(Sets.newHashSet(confLeafQueues)); queueMgr.updateAllocationConfiguration(allocConf); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestQueuePlacementPolicy.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestQueuePlacementPolicy.java index 3fe9ce31021..78242b28f92 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestQueuePlacementPolicy.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestQueuePlacementPolicy.java @@ -18,22 +18,28 @@ package org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair; import static org.junit.Assert.*; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; -import java.io.IOException; -import java.util.Collections; -import java.util.HashMap; -import java.util.HashSet; -import java.util.List; -import java.util.Map; -import java.util.Set; +import java.nio.charset.StandardCharsets; import javax.xml.parsers.DocumentBuilder; import javax.xml.parsers.DocumentBuilderFactory; import org.apache.commons.io.IOUtils; -import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.CommonConfigurationKeys; import org.apache.hadoop.security.GroupMappingServiceProvider; +import org.apache.hadoop.yarn.api.records.ApplicationId; +import org.apache.hadoop.yarn.api.records.ApplicationSubmissionContext; +import org.apache.hadoop.yarn.api.records.ContainerLaunchContext; +import org.apache.hadoop.yarn.api.records.Priority; +import org.apache.hadoop.yarn.api.records.Resource; +import org.apache.hadoop.yarn.exceptions.YarnException; +import org.apache.hadoop.yarn.server.resourcemanager.RMContext; +import org.apache.hadoop.yarn.server.resourcemanager.placement.ApplicationPlacementContext; +import org.apache.hadoop.yarn.server.resourcemanager.placement.PlacementManager; +import org.apache.hadoop.yarn.util.SystemClock; +import org.junit.After; import org.junit.Before; import org.junit.BeforeClass; import org.junit.Test; @@ -41,9 +47,17 @@ import org.w3c.dom.Element; public class TestQueuePlacementPolicy { - private final static Configuration conf = new Configuration(); - private Map> configuredQueues; - + private final static FairSchedulerConfiguration conf = + new FairSchedulerConfiguration(); + // Base setup needed, policy is an intermediate object + private PlacementManager placementManager; + private FairScheduler scheduler; + private QueueManager queueManager; + + // Locals used in each assignment + ApplicationSubmissionContext asc; + ApplicationPlacementContext context; + @BeforeClass public static void setup() { conf.setClass(CommonConfigurationKeys.HADOOP_SECURITY_GROUP_MAPPING, @@ -52,10 +66,27 @@ public static void setup() { @Before public void initTest() { - configuredQueues = new HashMap>(); - for (FSQueueType type : FSQueueType.values()) { - configuredQueues.put(type, new HashSet()); - } + SystemClock clock = SystemClock.getInstance(); + RMContext rmContext = mock(RMContext.class); + placementManager = new PlacementManager(); + scheduler = mock(FairScheduler.class); + when(scheduler.getClock()).thenReturn(clock); + when(scheduler.getRMContext()).thenReturn(rmContext); + when(scheduler.getConfig()).thenReturn(conf); + when(scheduler.getConf()).thenReturn(conf); + when(rmContext.getQueuePlacementManager()).thenReturn(placementManager); + AllocationConfiguration allocConf = new AllocationConfiguration(scheduler); + when(scheduler.getAllocationConfiguration()).thenReturn(allocConf); + queueManager = new QueueManager(scheduler); + queueManager.initialize(); + when(scheduler.getQueueManager()).thenReturn(queueManager); + } + + @After + public void cleanTest() { + placementManager = null; + queueManager = null; + scheduler = null; } @Test @@ -65,15 +96,18 @@ public void testSpecifiedUserPolicy() throws Exception { sb.append(" "); sb.append(" "); sb.append(""); - QueuePlacementPolicy policy = parse(sb.toString()); - assertEquals("root.specifiedq", - policy.assignAppToQueue("specifiedq", "someuser")); - assertEquals("root.someuser", - policy.assignAppToQueue("default", "someuser")); - assertEquals("root.otheruser", - policy.assignAppToQueue("default", "otheruser")); + createPolicy(sb.toString()); + + asc = newAppSubmissionContext("specifiedq"); + context = placementManager.placeApplication(asc, "someuser"); + assertEquals("root.specifiedq", context.getQueue()); + asc = newAppSubmissionContext("default"); + context = placementManager.placeApplication(asc, "someuser"); + assertEquals("root.someuser", context.getQueue()); + context = placementManager.placeApplication(asc, "otheruser"); + assertEquals("root.otheruser", context.getQueue()); } - + @Test public void testNoCreate() throws Exception { StringBuffer sb = new StringBuffer(); @@ -82,13 +116,22 @@ public void testNoCreate() throws Exception { sb.append(" "); sb.append(" "); sb.append(""); - - configuredQueues.get(FSQueueType.LEAF).add("root.someuser"); - QueuePlacementPolicy policy = parse(sb.toString()); - assertEquals("root.specifiedq", policy.assignAppToQueue("specifiedq", "someuser")); - assertEquals("root.someuser", policy.assignAppToQueue("default", "someuser")); - assertEquals("root.specifiedq", policy.assignAppToQueue("specifiedq", "otheruser")); - assertEquals("root.default", policy.assignAppToQueue("default", "otheruser")); + createPolicy(sb.toString()); + + createQueue(FSQueueType.LEAF, "root.someuser"); + + asc = newAppSubmissionContext("specifiedq"); + context = placementManager.placeApplication(asc, "someuser"); + assertEquals("root.specifiedq", context.getQueue()); + asc = newAppSubmissionContext("default"); + context = placementManager.placeApplication(asc, "someuser"); + assertEquals("root.someuser", context.getQueue()); + asc = newAppSubmissionContext("specifiedq"); + context = placementManager.placeApplication(asc, "otheruser"); + assertEquals("root.specifiedq", context.getQueue()); + asc = newAppSubmissionContext("default"); + context = placementManager.placeApplication(asc, "otheruser"); + assertEquals("root.default", context.getQueue()); } @Test @@ -98,63 +141,64 @@ public void testSpecifiedThenReject() throws Exception { sb.append(" "); sb.append(" "); sb.append(""); - QueuePlacementPolicy policy = parse(sb.toString()); - assertEquals("root.specifiedq", - policy.assignAppToQueue("specifiedq", "someuser")); - assertEquals(null, policy.assignAppToQueue("default", "someuser")); + createPolicy(sb.toString()); + asc = newAppSubmissionContext("specifiedq"); + context = placementManager.placeApplication(asc, "someuser"); + assertEquals("root.specifiedq", context.getQueue()); + asc = newAppSubmissionContext("default"); + context = placementManager.placeApplication(asc, "someuser"); + assertNull("Assignment should have been rejected and was not", context); } - @Test (expected = AllocationConfigurationException.class) - public void testOmittedTerminalRule() throws Exception { + @Test + public void testOmittedTerminalRule() { StringBuffer sb = new StringBuffer(); sb.append(""); sb.append(" "); sb.append(" "); sb.append(""); - parse(sb.toString()); + assertIfExceptionThrown(sb); } - @Test (expected = AllocationConfigurationException.class) - public void testTerminalRuleInMiddle() throws Exception { + @Test + public void testTerminalRuleInMiddle() { StringBuffer sb = new StringBuffer(); sb.append(""); sb.append(" "); sb.append(" "); sb.append(" "); sb.append(""); - parse(sb.toString()); + assertIfExceptionThrown(sb); } @Test - public void testTerminals() throws Exception { - // Should make it through without an exception + public void testTerminals() { + // The default rule is no longer considered terminal when the create flag + // is false. The throw now happens when configuring not when assigning the + // application StringBuffer sb = new StringBuffer(); sb.append(""); sb.append(" "); sb.append(" "); sb.append(""); - QueuePlacementPolicy policy = parse(sb.toString()); - try { - policy.assignAppToQueue("root.otherdefault", "user1"); - fail("Expect exception from having default rule with create=\'false\'"); - } catch (IllegalStateException se) { - } + assertIfExceptionThrown(sb); } @Test public void testDefaultRuleWithQueueAttribute() throws Exception { // This test covers the use case where we would like default rule // to point to a different queue by default rather than root.default - configuredQueues.get(FSQueueType.LEAF).add("root.someDefaultQueue"); + createQueue(FSQueueType.LEAF, "root.someDefaultQueue"); StringBuffer sb = new StringBuffer(); sb.append(""); sb.append(" "); sb.append(" "); sb.append(""); - QueuePlacementPolicy policy = parse(sb.toString()); - assertEquals("root.someDefaultQueue", - policy.assignAppToQueue("root.default", "user1")); + createPolicy(sb.toString()); + asc = newAppSubmissionContext("root.default"); + context = placementManager.placeApplication(asc, "user1"); + assertEquals("root.someDefaultQueue", context.getQueue()); } @Test @@ -180,12 +224,24 @@ public void testNestedUserQueueParsingErrors() { sb.append(""); assertIfExceptionThrown(sb); + + // If the parent rule does not have the create flag the nested rule is not + // terminal + sb = new StringBuffer(); + sb.append(""); + sb.append(" "); + sb.append(" "); + sb.append(" "); + sb.append(" "); + sb.append(""); + + assertIfExceptionThrown(sb); } private void assertIfExceptionThrown(StringBuffer sb) { Throwable th = null; try { - parse(sb.toString()); + createPolicy(sb.toString()); } catch (Exception e) { th = e; } @@ -201,17 +257,9 @@ public void testNestedUserQueueParsing() throws Exception { sb.append(" "); sb.append(" "); sb.append(" "); - sb.append(" "); sb.append(""); - Throwable th = null; - try { - parse(sb.toString()); - } catch (Exception e) { - th = e; - } - - assertNull(th); + createPolicy(sb.toString()); } @Test @@ -222,24 +270,26 @@ public void testNestedUserQueuePrimaryGroup() throws Exception { sb.append(" "); sb.append(" "); sb.append(" "); - sb.append(" "); sb.append(""); // User queue would be created under primary group queue - QueuePlacementPolicy policy = parse(sb.toString()); - assertEquals("root.user1group.user1", - policy.assignAppToQueue("root.default", "user1")); + createPolicy(sb.toString()); + asc = newAppSubmissionContext("root.default"); + context = placementManager.placeApplication(asc, "user1"); + assertEquals("root.user1group.user1", context.getQueue()); // Other rules above and below hierarchical user queue rule should work as // usual - configuredQueues.get(FSQueueType.LEAF).add("root.specifiedq"); + createQueue(FSQueueType.LEAF, "root.specifiedq"); // test if specified rule(above nestedUserQueue rule) works ok - assertEquals("root.specifiedq", - policy.assignAppToQueue("root.specifiedq", "user2")); - - // test if default rule(below nestedUserQueue rule) works - configuredQueues.get(FSQueueType.LEAF).add("root.user3group"); - assertEquals("root.default", - policy.assignAppToQueue("root.default", "user3")); + asc = newAppSubmissionContext("root.specifiedq"); + context = placementManager.placeApplication(asc, "user2"); + assertEquals("root.specifiedq", context.getQueue()); + + // Submit should fail if we cannot create the queue + createQueue(FSQueueType.LEAF, "root.user3group"); + asc = newAppSubmissionContext("root.default"); + context = placementManager.placeApplication(asc, "user3"); + assertNull("Submission should have failed and did not", context); } @Test @@ -253,18 +303,18 @@ public void testNestedUserQueuePrimaryGroupNoCreate() throws Exception { sb.append(" "); sb.append(""); - QueuePlacementPolicy policy = parse(sb.toString()); + createPolicy(sb.toString()); // Should return root.default since primary group 'root.user1group' is not // configured - assertEquals("root.default", - policy.assignAppToQueue("root.default", "user1")); + asc = newAppSubmissionContext("root.default"); + context = placementManager.placeApplication(asc, "user1"); + assertEquals("root.default", context.getQueue()); // Let's configure primary group and check if user queue is created - configuredQueues.get(FSQueueType.PARENT).add("root.user1group"); - policy = parse(sb.toString()); - assertEquals("root.user1group.user1", - policy.assignAppToQueue("root.default", "user1")); + createQueue(FSQueueType.PARENT, "root.user1group"); + context = placementManager.placeApplication(asc, "user1"); + assertEquals("root.user1group.user1", context.getQueue()); // Both Primary group and nestedUserQueue rule has create='false' sb = new StringBuffer(); @@ -277,16 +327,16 @@ public void testNestedUserQueuePrimaryGroupNoCreate() throws Exception { // Should return root.default since primary group and user queue for user 2 // are not configured. - assertEquals("root.default", - policy.assignAppToQueue("root.default", "user2")); + asc = newAppSubmissionContext("root.default"); + context = placementManager.placeApplication(asc, "user2"); + assertEquals("root.default", context.getQueue()); // Now configure both primary group and the user queue for user2 - configuredQueues.get(FSQueueType.PARENT).add("root.user2group"); - configuredQueues.get(FSQueueType.LEAF).add("root.user2group.user2"); - policy = parse(sb.toString()); + createQueue(FSQueueType.LEAF, "root.user2group.user2"); - assertEquals("root.user2group.user2", - policy.assignAppToQueue("root.default", "user2")); + // Try placing the same app again + context = placementManager.placeApplication(asc, "user2"); + assertEquals("root.user2group.user2", context.getQueue()); } @Test @@ -299,17 +349,18 @@ public void testNestedUserQueueSecondaryGroup() throws Exception { sb.append(" "); sb.append(""); - QueuePlacementPolicy policy = parse(sb.toString()); + createPolicy(sb.toString()); // Should return root.default since secondary groups are not configured - assertEquals("root.default", - policy.assignAppToQueue("root.default", "user1")); + asc = newAppSubmissionContext("root.default"); + context = placementManager.placeApplication(asc, "user1"); + assertEquals("root.default", context.getQueue()); // configure secondary group for user1 - configuredQueues.get(FSQueueType.PARENT).add("root.user1subgroup1"); - policy = parse(sb.toString()); + createQueue(FSQueueType.PARENT, "root.user1subgroup1"); + createPolicy(sb.toString()); // user queue created should be created under secondary group - assertEquals("root.user1subgroup1.user1", - policy.assignAppToQueue("root.default", "user1")); + context = placementManager.placeApplication(asc, "user1"); + assertEquals("root.user1subgroup1.user1", context.getQueue()); } @Test @@ -325,33 +376,66 @@ public void testNestedUserQueueSpecificRule() throws Exception { sb.append(""); // Let's create couple of parent queues - configuredQueues.get(FSQueueType.PARENT).add("root.parent1"); - configuredQueues.get(FSQueueType.PARENT).add("root.parent2"); - - QueuePlacementPolicy policy = parse(sb.toString()); - assertEquals("root.parent1.user1", - policy.assignAppToQueue("root.parent1", "user1")); - assertEquals("root.parent2.user2", - policy.assignAppToQueue("root.parent2", "user2")); + createQueue(FSQueueType.PARENT, "root.parent1"); + createQueue(FSQueueType.PARENT, "root.parent2"); + + createPolicy(sb.toString()); + asc = newAppSubmissionContext("root.parent1"); + context = placementManager.placeApplication(asc, "user1"); + assertEquals("root.parent1.user1", context.getQueue()); + asc = newAppSubmissionContext("root.parent2"); + context = placementManager.placeApplication(asc, "user2"); + assertEquals("root.parent2.user2", context.getQueue()); } @Test public void testNestedUserQueueDefaultRule() throws Exception { // This test covers the use case where we would like user queues to be // created under a default parent queue - configuredQueues.get(FSQueueType.PARENT).add("root.parentq"); StringBuffer sb = new StringBuffer(); sb.append(""); sb.append(" "); sb.append(" "); sb.append(" "); sb.append(" "); + sb.append(""); + + createPolicy(sb.toString()); + asc = newAppSubmissionContext("root.default"); + context = placementManager.placeApplication(asc, "user1"); + assertEquals("root.parentq.user1", context.getQueue()); + + // Same as above but now with the create flag false for the parent + createQueue(FSQueueType.PARENT, "root.parentq"); + sb = new StringBuffer(); + sb.append(""); + sb.append(" "); + sb.append(" "); + sb.append(" "); + sb.append(" "); sb.append(" "); sb.append(""); - QueuePlacementPolicy policy = parse(sb.toString()); - assertEquals("root.parentq.user1", - policy.assignAppToQueue("root.default", "user1")); + createPolicy(sb.toString()); + asc = newAppSubmissionContext("root.default"); + context = placementManager.placeApplication(asc, "user1"); + assertEquals("root.parentq.user1", context.getQueue()); + + // Parent queue returned is already a configured LEAF, should fail and the + // context is null. + createQueue(FSQueueType.LEAF, "root.parentq"); + sb = new StringBuffer(); + sb.append(""); + sb.append(" "); + sb.append(" "); + sb.append(" "); + sb.append(" "); + sb.append(""); + + createPolicy(sb.toString()); + asc = newAppSubmissionContext("root.default"); + context = placementManager.placeApplication(asc, "user1"); + assertNull("Submission should have failed and did not", context); } @Test @@ -361,9 +445,10 @@ public void testUserContainsPeriod() throws Exception { sb.append(""); sb.append(" "); sb.append(""); - QueuePlacementPolicy policy = parse(sb.toString()); - assertEquals("root.first_dot_last", - policy.assignAppToQueue("default", "first.last")); + createPolicy(sb.toString()); + asc = newAppSubmissionContext("default"); + context = placementManager.placeApplication(asc, "first.last"); + assertEquals("root.first_dot_last", context.getQueue()); sb = new StringBuffer(); sb.append(""); @@ -371,11 +456,14 @@ public void testUserContainsPeriod() throws Exception { sb.append(" "); sb.append(" "); sb.append(" "); - sb.append(" "); sb.append(""); - policy = parse(sb.toString()); - assertEquals("root.default.first_dot_last", - policy.assignAppToQueue("root.default", "first.last")); + // specified create is false, bypass the rule + // default rule has create which requires a PARENT queue: remove the LEAF + queueManager.removeLeafQueue("root.default"); + createPolicy(sb.toString()); + asc = newAppSubmissionContext("root.default"); + context = placementManager.placeApplication(asc, "first_dot_last"); + assertEquals("root.default.first_dot_last", context.getQueue()); } @Test @@ -386,22 +474,23 @@ public void testGroupContainsPeriod() throws Exception { sb.append(" "); sb.append(" "); sb.append(" "); - sb.append(" "); + //sb.append(" "); sb.append(""); conf.setClass(CommonConfigurationKeys.HADOOP_SECURITY_GROUP_MAPPING, PeriodGroupsMapping.class, GroupMappingServiceProvider.class); // User queue would be created under primary group queue, and the period // in the group name should be converted into _dot_ - QueuePlacementPolicy policy = parse(sb.toString()); - assertEquals("root.user1_dot_group.user1", - policy.assignAppToQueue("root.default", "user1")); + createPolicy(sb.toString()); + asc = newAppSubmissionContext("root.default"); + context = placementManager.placeApplication(asc, "user1"); + assertEquals("root.user1_dot_group.user1", context.getQueue()); conf.setClass(CommonConfigurationKeys.HADOOP_SECURITY_GROUP_MAPPING, SimpleGroupsMapping.class, GroupMappingServiceProvider.class); } - @Test(expected=IOException.class) + @Test(expected=YarnException.class) public void testEmptyGroupsPrimaryGroupRule() throws Exception { StringBuffer sb = new StringBuffer(); sb.append(""); @@ -412,18 +501,49 @@ public void testEmptyGroupsPrimaryGroupRule() throws Exception { // Add a static mapping that returns empty groups for users conf.setStrings(CommonConfigurationKeys .HADOOP_USER_GROUP_STATIC_OVERRIDES, "emptygroupuser="); - QueuePlacementPolicy policy = parse(sb.toString()); - policy.assignAppToQueue(null, "emptygroupuser"); + createPolicy(sb.toString()); + asc = newAppSubmissionContext("root.fake"); + context = placementManager.placeApplication(asc, "emptygroupuser"); } - private QueuePlacementPolicy parse(String str) throws Exception { + private void createPolicy(String str) + throws AllocationConfigurationException { // Read and parse the allocations file. - DocumentBuilderFactory docBuilderFactory = DocumentBuilderFactory - .newInstance(); - docBuilderFactory.setIgnoringComments(true); - DocumentBuilder builder = docBuilderFactory.newDocumentBuilder(); - Document doc = builder.parse(IOUtils.toInputStream(str)); - Element root = doc.getDocumentElement(); - return QueuePlacementPolicy.fromXml(root, configuredQueues, conf); + Element root = null; + try { + DocumentBuilderFactory docBuilderFactory = DocumentBuilderFactory + .newInstance(); + docBuilderFactory.setIgnoringComments(true); + DocumentBuilder builder = docBuilderFactory.newDocumentBuilder(); + Document doc = builder.parse(IOUtils.toInputStream(str, + StandardCharsets.UTF_8)); + root = doc.getDocumentElement(); + } catch (Exception ex) { + // Don't really want to test the xml parsing side, + // let it fail with a null config below. + } + QueuePlacementPolicy.fromXml(root, scheduler); + } + + private ApplicationSubmissionContext newAppSubmissionContext(String queue) { + ApplicationId appId = ApplicationId.newInstance(1L, 1); + Priority prio = Priority.UNDEFINED; + Resource resource = Resource.newInstance(1, 1); + ContainerLaunchContext amContainer = + ContainerLaunchContext.newInstance(null, null, null, null, null, null); + return ApplicationSubmissionContext.newInstance(appId, "test", queue, + prio, amContainer, false, false, 1, resource, "testing"); + } + + private void createQueue(FSQueueType type, String name) { + // Create a queue as if it is in the config. + FSQueue queue = queueManager.createQueue(name, type); + assertNotNull("Queue not created", queue); + // walk up the list till we have a non dynamic queue + // root is always non dynamic + do { + queue.setDynamic(false); + queue = queue.parent; + } while (queue.isDynamic()); } } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestSchedulingPolicy.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestSchedulingPolicy.java index b016c1b4fb8..e9075c9a8a3 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestSchedulingPolicy.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestSchedulingPolicy.java @@ -30,6 +30,8 @@ import org.apache.commons.logging.LogFactory; import org.apache.hadoop.yarn.api.records.Priority; import org.apache.hadoop.yarn.api.records.Resource; +import org.apache.hadoop.yarn.server.resourcemanager.RMContext; +import org.apache.hadoop.yarn.server.resourcemanager.placement.PlacementManager; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.policies.DominantResourceFairnessPolicy; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.policies.FairSharePolicy; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.policies.FifoPolicy; @@ -40,6 +42,8 @@ import static org.junit.Assert.assertNull; import static org.junit.Assert.assertTrue; import static org.junit.Assert.assertNotNull; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; public class TestSchedulingPolicy { private static final Log LOG = LogFactory.getLog(TestSchedulingPolicy.class); @@ -52,6 +56,11 @@ public void setUp() throws Exception { scheduler = new FairScheduler(); conf = new FairSchedulerConfiguration(); + // since this runs outside of the normal context we need to set one + RMContext rmContext = mock(RMContext.class); + PlacementManager placementManager = new PlacementManager(); + when(rmContext.getQueuePlacementManager()).thenReturn(placementManager); + scheduler.setRMContext(rmContext); } public void testParseSchedulingPolicy() diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/TestFairSchedulerQueueInfo.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/TestFairSchedulerQueueInfo.java index 83b7e41112f..454609b5efb 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/TestFairSchedulerQueueInfo.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/TestFairSchedulerQueueInfo.java @@ -19,6 +19,8 @@ package org.apache.hadoop.yarn.server.resourcemanager.webapp.dao; import org.apache.hadoop.yarn.api.records.Resource; +import org.apache.hadoop.yarn.server.resourcemanager.RMContext; +import org.apache.hadoop.yarn.server.resourcemanager.placement.PlacementManager; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.AllocationConfiguration; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.FSQueue; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.FairScheduler; @@ -37,19 +39,25 @@ public class TestFairSchedulerQueueInfo { @Test - public void testEmptyChildQueues() throws Exception { - FairSchedulerConfiguration conf = new FairSchedulerConfiguration(); + public void testEmptyChildQueues() { + FairSchedulerConfiguration fsConf = new FairSchedulerConfiguration(); + RMContext rmContext = mock(RMContext.class); + PlacementManager placementManager = new PlacementManager(); + SystemClock clock = SystemClock.getInstance(); FairScheduler scheduler = mock(FairScheduler.class); - AllocationConfiguration allocConf = new AllocationConfiguration(conf); - when(scheduler.getAllocationConfiguration()).thenReturn(allocConf); - when(scheduler.getConf()).thenReturn(conf); - when(scheduler.getClusterResource()).thenReturn(Resource.newInstance(1, 1)); + when(scheduler.getConf()).thenReturn(fsConf); + when(scheduler.getConfig()).thenReturn(fsConf); + when(scheduler.getRMContext()).thenReturn(rmContext); + when(rmContext.getQueuePlacementManager()).thenReturn(placementManager); + when(scheduler.getClusterResource()).thenReturn( + Resource.newInstance(1, 1)); when(scheduler.getResourceCalculator()).thenReturn( new DefaultResourceCalculator()); - SystemClock clock = SystemClock.getInstance(); when(scheduler.getClock()).thenReturn(clock); + AllocationConfiguration allocConf = new AllocationConfiguration(scheduler); + when(scheduler.getAllocationConfiguration()).thenReturn(allocConf); QueueManager queueManager = new QueueManager(scheduler); - queueManager.initialize(conf); + queueManager.initialize(); FSQueue testQueue = queueManager.getLeafQueue("test", true); FairSchedulerQueueInfo queueInfo =