diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/placement/DefaultPlacementRule.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/placement/DefaultPlacementRule.java new file mode 100644 index 00000000000..bdb267595a3 --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/placement/DefaultPlacementRule.java @@ -0,0 +1,160 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.yarn.server.resourcemanager.placement; + +import com.google.common.annotations.VisibleForTesting; +import org.apache.hadoop.classification.InterfaceAudience; +import org.apache.hadoop.classification.InterfaceStability; +import org.apache.hadoop.yarn.api.records.ApplicationSubmissionContext; +import org.apache.hadoop.yarn.conf.YarnConfiguration; +import org.apache.hadoop.yarn.server.resourcemanager.scheduler.ResourceScheduler; +import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.FSQueue; +import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.FairScheduler; +import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.QueueManager; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.w3c.dom.Element; + +import java.io.IOException; + +import static org.apache.hadoop.yarn.server.resourcemanager.placement.FairQueuePlacementUtils.assureRoot; +import static org.apache.hadoop.yarn.server.resourcemanager.placement.FairQueuePlacementUtils.isValidQueueName; + +/** + * Places apps in the specified default queue. If no default queue is + * specified the app is placed in root.default queue. + */ +@InterfaceAudience.Private +@InterfaceStability.Unstable +public class DefaultPlacementRule extends PlacementRule { + private static final Logger LOG = + LoggerFactory.getLogger(DefaultPlacementRule.class); + + private QueueManager queueManager; + @VisibleForTesting + public String defaultQueueName; + + /** + * Set the config based on the type of object passed in. + * @param initArg the config to be set + */ + @Override + public void setConfig(Object initArg) { + if (null == initArg) { + LOG.debug("Null object passed in: no config set"); + return; + } + if (initArg instanceof Element) { + LOG.debug("Setting config from XML"); + setConfig(Element.class.cast(initArg)); + } else { + if (initArg instanceof Boolean) { + LOG.debug("Setting config from Boolean"); + setConfig(Boolean.class.cast(initArg)); + } else { + LOG.info("Unknown object type passed in as config for rule {}: {}", + getName(), initArg.getClass()); + } + } + } + + /** + * Set the rule config from the xml config. + * @param conf An xml element from the {@link FairScheduler#conf} + */ + public void setConfig(Element conf) { + // Default create for this rule is true + createQueue = true; + // No config can be set when no policy is defined and we use defaults + if (conf != null) { + String create = conf.getAttribute("create"); + if (create != null && !create.isEmpty()) { + createQueue = Boolean.parseBoolean(create); + } + defaultQueueName = conf.getAttribute("queue"); + // A queue read from the config could be illegal check it: fall back to + // the config default if it is the case + // However we cannot clean the name as a nested name is allowed. + if (!isValidQueueName(defaultQueueName)) { + LOG.error("Default rule configured with an illegal queue name: '{}'", + defaultQueueName); + defaultQueueName = null; + } + } + // The queue name does not have to be set and we really use "default" + if (defaultQueueName == null || defaultQueueName.isEmpty()) { + defaultQueueName = assureRoot(YarnConfiguration.DEFAULT_QUEUE_NAME); + } else { + defaultQueueName = assureRoot(defaultQueueName); + } + LOG.debug("Default rule instantiated with queue name: {}, " + + "and create flag: {}", defaultQueueName, createQueue); + } + + /** + * Set the rule config just setting the create flag. + * @param create flag to allow queue creation for this rule + */ + public void setConfig(Boolean create) { + createQueue = create; + // No config so fall back to the real default. + defaultQueueName = assureRoot(YarnConfiguration.DEFAULT_QUEUE_NAME); + LOG.debug("Default rule instantiated with default queue name: {}, " + + "and create flag: {}", defaultQueueName, createQueue); + } + + @Override + public boolean initialize(ResourceScheduler scheduler) throws IOException { + if (!(scheduler instanceof FairScheduler)) { + throw new IOException( + "Default rule can only be configured for the FairScheduler"); + } + if (getParentRule() != null) { + throw new IOException( + "Parent rule should not be configured for Default rule."); + } + + FairScheduler fs = (FairScheduler) scheduler; + queueManager = fs.getQueueManager(); + + return true; + } + + @Override + public ApplicationPlacementContext getPlacementForApp( + ApplicationSubmissionContext asc, String user) { + + // If we can create the queue in the rule or the queue exists return it + if (createQueue || configuredQueue(defaultQueueName)) { + return new ApplicationPlacementContext(defaultQueueName); + } + return null; + } + + /** + * Check if the queue exists and is part of the configuration i.e. not + * a {@link FSQueue#isDynamic()} queue. + * @param queueName name of the queue to check + * @return true if the queue exists and is a "configured" queue + */ + private boolean configuredQueue(String queueName) { + FSQueue queue = queueManager.getQueue(queueName); + return (queue != null && !queue.isDynamic()); + } +} diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/placement/FairQueuePlacementUtils.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/placement/FairQueuePlacementUtils.java new file mode 100644 index 00000000000..5663695e399 --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/placement/FairQueuePlacementUtils.java @@ -0,0 +1,100 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.yarn.server.resourcemanager.placement; + +import org.apache.hadoop.classification.InterfaceAudience; +import org.apache.hadoop.classification.InterfaceStability; +import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.FairSchedulerUtilities; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +/** + * Utility methods used by Fair scheduler placement rules. + * {@link + * org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.FairScheduler} + */ +@InterfaceAudience.Private +@InterfaceStability.Unstable +public final class FairQueuePlacementUtils { + private static final Logger LOG = + LoggerFactory.getLogger(FairQueuePlacementUtils.class); + + // Constants for name clean up and hierarchy checks + protected static final String DOT = "."; + protected static final String DOT_REPLACEMENT = "_dot_"; + protected static final String ROOT_QUEUE = "root"; + + private FairQueuePlacementUtils() { + } + + /** + * Replace the periods in the username or group name with "_dot_" and + * remove trailing and leading whitespace. + * + * @param name The name to clean + * @return The name with {@link #DOT} replaced with {@link #DOT_REPLACEMENT} + */ + protected static String cleanName(String name) { + name = FairSchedulerUtilities.trimQueueName(name); + if (name.contains(DOT)) { + String converted = name.replaceAll("\\.", DOT_REPLACEMENT); + LOG.warn("Name {} is converted to {} when it is used as a queue name.", + name, converted); + return converted; + } else { + return name; + } + } + + /** + * Assure root prefix for a queue name. + * + * @param queueName The queue name to check for the root prefix + * @return The root prefixed queue name + */ + protected static String assureRoot(String queueName) { + if (queueName != null && !queueName.isEmpty()) { + if (!queueName.startsWith(ROOT_QUEUE + DOT) && + !queueName.equals(ROOT_QUEUE)) { + queueName = ROOT_QUEUE + DOT + queueName; + } + } else { + LOG.warn("AssureRoot: queueName is empty or null."); + } + return queueName; + } + + /** + * Validate the queue name: it may not start or end with a {@link #DOT}. + * + * @param queueName The queue name to validate + * @return false if the queue name starts or ends with a + * {@link #DOT}, true + */ + protected static boolean isValidQueueName(String queueName) { + if (queueName != null) { + if (queueName.equals(FairSchedulerUtilities.trimQueueName(queueName)) && + !queueName.startsWith(DOT) && + !queueName.endsWith(DOT)) { + return true; + } + } + return false; + } +} diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/placement/PlacementFactory.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/placement/PlacementFactory.java index a5de66463bb..10df789d3da 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/placement/PlacementFactory.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/placement/PlacementFactory.java @@ -20,12 +20,16 @@ import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; +import org.apache.hadoop.classification.InterfaceAudience; +import org.apache.hadoop.classification.InterfaceStability; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.util.ReflectionUtils; /** * Factory class for creating instances of {@link PlacementRule}. */ +@InterfaceAudience.Private +@InterfaceStability.Unstable public final class PlacementFactory { private static final Log LOG = LogFactory.getLog(PlacementFactory.class); @@ -34,6 +38,14 @@ private PlacementFactory() { // Unused. } + /** + * Create a new {@link PlacementRule} based on the rule class from the + * configuration. This is used to instantiate rules by the FairScheduler + * which resolve the class before this call. + * @param ruleStr The name of the class to instantiate + * @param conf The configuration object to set for the rule + * @return Created class instance + */ public static PlacementRule getPlacementRule(String ruleStr, Configuration conf) throws ClassNotFoundException { @@ -42,4 +54,20 @@ public static PlacementRule getPlacementRule(String ruleStr, LOG.info("Using PlacementRule implementation - " + ruleClass); return ReflectionUtils.newInstance(ruleClass, conf); } + + /** + * Create a new {@link PlacementRule} based on the rule class from the + * configuration. This is used to instantiate rules by the FairScheduler + * which resolve the class before this call. + * @param ruleClass The specific class reference to instantiate + * @param initArg The config to set + * @return Created class instance + */ + public static PlacementRule getPlacementRule( + Class ruleClass, Object initArg) { + LOG.info("Creating PlacementRule implementation: " + ruleClass); + PlacementRule rule = ReflectionUtils.newInstance(ruleClass, null); + rule.setConfig(initArg); + return rule; + } } \ No newline at end of file diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/placement/PlacementRule.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/placement/PlacementRule.java index 0f3d43c5ad7..83468eaac21 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/placement/PlacementRule.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/placement/PlacementRule.java @@ -20,35 +20,85 @@ import java.io.IOException; +import com.google.common.annotations.VisibleForTesting; +import org.apache.hadoop.classification.InterfaceAudience; +import org.apache.hadoop.classification.InterfaceStability; import org.apache.hadoop.yarn.api.records.ApplicationSubmissionContext; import org.apache.hadoop.yarn.exceptions.YarnException; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.ResourceScheduler; +/** + * Abstract base for all Placement Rules. + */ +@InterfaceAudience.Private +@InterfaceStability.Unstable public abstract class PlacementRule { + // Flag to show if the rule can create a queue + @VisibleForTesting + public boolean createQueue = false; + private PlacementRule parentRule = null; + + /** + * Set a rule to generate the parent queue dynamically. + * @param parent A PlacementRule + */ + public void setParentRule(PlacementRule parent) { + this.parentRule = parent; + } + + /** + * Get the rule that is set to generate the parent queue dynamically. + * @return The rule set or null if not set. + */ + public PlacementRule getParentRule() { + return parentRule; + } + /** + * Set the config based on the passed in argument. + * This is used only by the FairScheduler to set the configuration from the + * XML element or a simple boolean flag. This construct is used to not + * pollute this abstract class with implementation specific references. + */ + public void setConfig(Object initArg) { + // Default is a noop + } + + /** + * Return the name of the rule. + * @return The name of the rule, the fully qualified class name. + */ public String getName() { return this.getClass().getName(); } - public abstract boolean initialize( - ResourceScheduler scheduler) throws IOException; + /** + * Initialize the rule with the scheduler. + * @param scheduler the scheduler using the rule + * @return true or false The outcome of the + * initialisation, rule dependent response which might not be persisted in + * the rule. + * @throws IOException for any errors + */ + public abstract boolean initialize(ResourceScheduler scheduler) + throws IOException; /** - * Get queue for a given application - * - * @param asc application submission context - * @param user userName + * Return the scheduler queue name the application should be placed in + * wrapped in an {@link ApplicationPlacementContext} object. + * + * A non null return value places the application in a queue, + * a null value means the queue is not yet determined. The + * next {@link PlacementRule} in the list maintained in the + * {@link PlacementManager} will be executed. + * + * @param asc The context of the application created on submission + * @param user The name of the user submitting the application * - * @throws YarnException - * if any error happens + * @throws YarnException for any error while executing the rule * - * @return

- * non-null value means it is determined - *

- *

- * null value means it is undetermined, so next {@link PlacementRule} - * in the {@link PlacementManager} will take care - *

+ * @return The queue name wrapped in {@link ApplicationPlacementContext} or + * null in no queue was resolved */ public abstract ApplicationPlacementContext getPlacementForApp( ApplicationSubmissionContext asc, String user) throws YarnException; diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/placement/PrimaryGroupPlacementRule.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/placement/PrimaryGroupPlacementRule.java new file mode 100644 index 00000000000..ce000103c22 --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/placement/PrimaryGroupPlacementRule.java @@ -0,0 +1,180 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.yarn.server.resourcemanager.placement; + +import org.apache.hadoop.classification.InterfaceAudience; +import org.apache.hadoop.classification.InterfaceStability; +import org.apache.hadoop.security.Groups; +import org.apache.hadoop.yarn.api.records.ApplicationSubmissionContext; +import org.apache.hadoop.yarn.exceptions.YarnException; +import org.apache.hadoop.yarn.server.resourcemanager.scheduler.ResourceScheduler; +import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.FSLeafQueue; +import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.FSQueue; +import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.FairScheduler; +import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.QueueManager; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.w3c.dom.Element; + +import java.io.IOException; +import java.util.List; + +import static org.apache.hadoop.yarn.server.resourcemanager.placement.FairQueuePlacementUtils.DOT; +import static org.apache.hadoop.yarn.server.resourcemanager.placement.FairQueuePlacementUtils.assureRoot; +import static org.apache.hadoop.yarn.server.resourcemanager.placement.FairQueuePlacementUtils.cleanName; + +/** + * Places apps in queues by the primary group of the submitter. + */ +@InterfaceAudience.Private +@InterfaceStability.Unstable +public class PrimaryGroupPlacementRule extends PlacementRule { + private static final Logger LOG = + LoggerFactory.getLogger(PrimaryGroupPlacementRule.class); + + private QueueManager queueManager; + private Groups groupProvider; + + /** + * Set the config based on the type of object passed in. + * @param initArg the config to be set + */ + @Override + public void setConfig(Object initArg) { + if (null == initArg) { + LOG.debug("Null object passed in: no config set"); + return; + } + if (initArg instanceof Element) { + LOG.debug("Setting config from XML"); + setConfig(Element.class.cast(initArg)); + } else { + if (initArg instanceof Boolean) { + LOG.debug("Setting config from Boolean"); + setConfig(Boolean.class.cast(initArg)); + } else { + LOG.info("Unknown object type passed in as config for rule {}: {}", + getName(), initArg.getClass()); + } + } + } + + /** + * Set the rule config from the xml config. + * @param conf An xml element from the {@link FairScheduler#conf} + */ + public void setConfig(Element conf) { + // No config can be set when no policy is defined and we use defaults + createQueue = true; + if (conf != null) { + String create = conf.getAttribute("create"); + if (create != null && !create.isEmpty()) { + createQueue = Boolean.parseBoolean(create); + } + } + LOG.debug("PrimaryGroupPlacementRule instantiated with create flag: {}", + createQueue); + } + + /** + * Set the rule config just setting the create flag. + * @param create flag to allow queue creation for this rule + */ + public void setConfig(Boolean create) { + createQueue = create.booleanValue(); + LOG.debug("PrimaryGroupPlacementRule instantiated with create flag: {}", + createQueue); + } + + @Override + public boolean initialize(ResourceScheduler scheduler) throws IOException { + if (!(scheduler instanceof FairScheduler)) { + throw new IOException( + "PrimaryGroup rule can only be configured for the FairScheduler"); + } + if (getParentRule() != null && + getParentRule().getName().equals(getName())) { + throw new IOException("Parent rule should not be the same type as the " + + "child rule (PrimaryGroup)"); + } + + FairScheduler fs = (FairScheduler) scheduler; + queueManager = fs.getQueueManager(); + groupProvider = new Groups(fs.getConfig()); + + return true; + } + + @Override + public ApplicationPlacementContext getPlacementForApp( + ApplicationSubmissionContext asc, String user) throws YarnException { + + // All users should have at least one group the primary group. If no groups + // are returned then there is a real issue. + final List groupList; + try { + groupList = groupProvider.getGroups(user); + } catch (IOException ioe) { + throw new YarnException("Group resolution failed", ioe); + } + if (groupList.isEmpty()) { + LOG.error("Group placement rule failed: No groups returned for user {}", + user); + throw new YarnException("No groups returned for user " + user); + } + + String cleanGroup = cleanName(groupList.get(0)); + String queueName; + PlacementRule parentRule = getParentRule(); + + if (getParentRule() != null) { + LOG.debug("PrimaryGroup rule: parent rule found: {}", + parentRule.getName()); + ApplicationPlacementContext parent = + parentRule.getPlacementForApp(asc, user); + if (parent == null || + queueManager.getQueue(parent.getQueue()) instanceof FSLeafQueue) { + LOG.debug("PrimaryGroup rule: parent rule failed"); + return null; + } + LOG.debug("PrimaryGroup rule: parent rule result: {}", + parent.getQueue()); + queueName = parent.getQueue() + DOT + cleanGroup; + } else { + queueName = assureRoot(cleanGroup); + } + + // If we can create the queue in the rule or the queue exists return it + if (createQueue || configuredQueue(queueName)) { + return new ApplicationPlacementContext(queueName); + } + return null; + } + + /** + * Check if the queue exists and is part of the configuration i.e. not + * a {@link FSQueue#isDynamic()} queue. + * @param queueName name of the queue to check + * @return true if the queue exists and is a "configured" queue + */ + private boolean configuredQueue(String queueName) { + FSQueue queue = queueManager.getQueue(queueName); + return (queue != null && !queue.isDynamic()); + } +} diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/placement/RejectPlacementRule.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/placement/RejectPlacementRule.java new file mode 100644 index 00000000000..6e43428ffef --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/placement/RejectPlacementRule.java @@ -0,0 +1,69 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.yarn.server.resourcemanager.placement; + +import org.apache.hadoop.classification.InterfaceAudience; +import org.apache.hadoop.classification.InterfaceStability; +import org.apache.hadoop.yarn.api.records.ApplicationSubmissionContext; +import org.apache.hadoop.yarn.server.resourcemanager.scheduler.ResourceScheduler; +import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.FairScheduler; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.w3c.dom.Element; + +import java.io.IOException; + +/** + * Rejects all placements. + */ +@InterfaceAudience.Private +@InterfaceStability.Unstable +public class RejectPlacementRule extends PlacementRule { + private static final Logger LOG = + LoggerFactory.getLogger(RejectPlacementRule.class); + + /** + * Set the config based on the type of object passed in. + * @param initArg the config to be set + */ + @Override + public void setConfig(Object initArg) { + // This rule ignores all config, just log and return + LOG.debug("RejectPlacementRule instantiated"); + } + + @Override + public boolean initialize(ResourceScheduler scheduler) throws IOException { + if (!(scheduler instanceof FairScheduler)) { + throw new IOException( + "Reject rule can only be configured for the FairScheduler"); + } + if (getParentRule() != null) { + throw new IOException( + "Parent rule should not be configured for Reject rule."); + } + return true; + } + + @Override + public ApplicationPlacementContext getPlacementForApp( + ApplicationSubmissionContext asc, String user) { + return null; + } +} diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/placement/SecondaryGroupExistingPlacementRule.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/placement/SecondaryGroupExistingPlacementRule.java new file mode 100644 index 00000000000..bfb97b8daaa --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/placement/SecondaryGroupExistingPlacementRule.java @@ -0,0 +1,179 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.yarn.server.resourcemanager.placement; + +import org.apache.hadoop.classification.InterfaceAudience; +import org.apache.hadoop.classification.InterfaceStability; +import org.apache.hadoop.security.Groups; +import org.apache.hadoop.yarn.api.records.ApplicationSubmissionContext; +import org.apache.hadoop.yarn.exceptions.YarnException; +import org.apache.hadoop.yarn.server.resourcemanager.scheduler.ResourceScheduler; +import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.FSLeafQueue; +import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.FSQueue; +import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.FairScheduler; +import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.QueueManager; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.w3c.dom.Element; + +import java.io.IOException; +import java.util.List; + +import static org.apache.hadoop.yarn.server.resourcemanager.placement.FairQueuePlacementUtils.DOT; +import static org.apache.hadoop.yarn.server.resourcemanager.placement.FairQueuePlacementUtils.assureRoot; +import static org.apache.hadoop.yarn.server.resourcemanager.placement.FairQueuePlacementUtils.cleanName; + +/** + * Places apps in queues by the secondary group of the submitter, if the + * submitter is a member of more than one group. + * The first "matching" queue based on the group list is returned. The match + * takes into account the parent rule and create flag, + */ +@InterfaceAudience.Private +@InterfaceStability.Unstable +public class SecondaryGroupExistingPlacementRule extends PlacementRule { + private static final Logger LOG = + LoggerFactory.getLogger(SecondaryGroupExistingPlacementRule.class); + + private QueueManager queueManager; + private Groups groupProvider; + + /** + * Set the config based on the type of object passed in. + * @param initArg the config to be set + */ + @Override + public void setConfig(Object initArg) { + if (null == initArg) { + LOG.debug("Null object passed in: no config set"); + return; + } + if (initArg instanceof Element) { + LOG.debug("Setting config from XML"); + setConfig(Element.class.cast(initArg)); + } else { + if (initArg instanceof Boolean) { + LOG.debug("Setting config from Boolean"); + setConfig(Boolean.class.cast(initArg)); + } else { + LOG.info("Unknown object type passed in as config for rule {}: {}", + getName(), initArg.getClass()); + } + } + } + + /** + * Set the rule config from the xml config. + * @param conf An xml element from the {@link FairScheduler#conf} + */ + public void setConfig(Element conf) { + // No config can be set when no policy is defined and we use defaults + createQueue = true; + if (conf != null) { + String create = conf.getAttribute("create"); + if (create != null && !create.isEmpty()) { + createQueue = Boolean.parseBoolean(create); + } + } + LOG.debug("SecondaryGroupExistingPlacementRule instantiated with create " + + "flag: {}", createQueue); + } + + /** + * Set the rule config just setting the create flag. + * @param create flag to allow queue creation for this rule + */ + public void setConfig(Boolean create) { + createQueue = create.booleanValue(); + LOG.debug("SecondaryGroupExistingPlacementRule instantiated with create " + + "flag: {}", createQueue); + } + + @Override + public boolean initialize(ResourceScheduler scheduler) throws IOException { + if (!(scheduler instanceof FairScheduler)) { + throw new IOException("SecondaryGroupExisting rule can only be " + + "configured for the FairScheduler"); + } + if (getParentRule() != null && + getParentRule().getName().equals(getName())) { + throw new IOException("Parent rule should not be the same type as the " + + "child rule (SecondaryGroupExisting)"); + } + + FairScheduler fs = (FairScheduler) scheduler; + queueManager = fs.getQueueManager(); + groupProvider = new Groups(fs.getConfig()); + + return true; + } + + @Override + public ApplicationPlacementContext getPlacementForApp( + ApplicationSubmissionContext asc, String user) throws YarnException { + + // All users should have at least one group the primary group. If no groups + // are returned then there is a real issue. + final List groupList; + try { + groupList = groupProvider.getGroups(user); + } catch (IOException ioe) { + throw new YarnException("Group resolution failed", ioe); + } + + String parentQueue = null; + PlacementRule parentRule = getParentRule(); + + if (parentRule != null) { + LOG.debug("SecondaryGroupExisting rule: parent rule found: {}", + parentRule.getName()); + ApplicationPlacementContext parent = + parentRule.getPlacementForApp(asc, user); + if (parent == null || + queueManager.getQueue(parent.getQueue()) instanceof FSLeafQueue) { + LOG.debug("SecondaryGroupExisting rule: parent rule failed"); + return null; + } + parentQueue = parent.getQueue(); + LOG.debug("SecondaryGroupExisting rule: parent rule result: {}", + parentQueue); + } + // now check the groups inside the parent + for (int i = 1; i < groupList.size(); i++) { + String group = cleanName(groupList.get(i)); + String queueName = + parentQueue == null ? assureRoot(group) : parentQueue + DOT + group; + if (configuredQueue(queueName)) { + return new ApplicationPlacementContext(queueName); + } + } + return null; + } + + /** + * Check if the queue exists and is part of the configuration i.e. not + * a {@link FSQueue#isDynamic()} queue. + * @param queueName name of the queue to check + * @return true if the queue exists and is a "configured" queue + */ + private boolean configuredQueue(String queueName) { + FSQueue queue = queueManager.getQueue(queueName); + return (queue != null && !queue.isDynamic()); + } +} diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/placement/SpecifiedPlacementRule.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/placement/SpecifiedPlacementRule.java new file mode 100644 index 00000000000..764f0aaff1c --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/placement/SpecifiedPlacementRule.java @@ -0,0 +1,152 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.yarn.server.resourcemanager.placement; + +import org.apache.hadoop.classification.InterfaceAudience; +import org.apache.hadoop.classification.InterfaceStability; +import org.apache.hadoop.yarn.api.records.ApplicationSubmissionContext; +import org.apache.hadoop.yarn.conf.YarnConfiguration; +import org.apache.hadoop.yarn.exceptions.YarnException; +import org.apache.hadoop.yarn.server.resourcemanager.scheduler.ResourceScheduler; +import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.FSQueue; +import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.FairScheduler; +import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.QueueManager; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.w3c.dom.Element; + +import java.io.IOException; + +import static org.apache.hadoop.yarn.server.resourcemanager.placement.FairQueuePlacementUtils.assureRoot; +import static org.apache.hadoop.yarn.server.resourcemanager.placement.FairQueuePlacementUtils.isValidQueueName; + +/** + * Places apps in queues by requested queue of the submitter. + */ +@InterfaceAudience.Private +@InterfaceStability.Unstable +public class SpecifiedPlacementRule extends PlacementRule { + private static final Logger LOG = + LoggerFactory.getLogger(SpecifiedPlacementRule.class); + + private QueueManager queueManager; + + /** + * Set the config based on the type of object passed in. + * @param initArg the config to be set + */ + @Override + public void setConfig(Object initArg) { + if (null == initArg) { + LOG.debug("Null object passed in: no config set"); + return; + } + if (initArg instanceof Element) { + LOG.debug("Setting config from XML"); + setConfig(Element.class.cast(initArg)); + } else { + if (initArg instanceof Boolean) { + LOG.debug("Setting config from Boolean"); + setConfig(Boolean.class.cast(initArg)); + } else { + LOG.info("Unknown object type passed in as config for rule {}: {}", + getName(), initArg.getClass()); + } + } + } + + /** + * Set the rule config from the xml config. + * @param conf An xml element from the {@link FairScheduler#conf} + */ + public void setConfig(Element conf) { + createQueue = true; + // No config can be set when no policy is defined and we use defaults + if (conf != null) { + String create = conf.getAttribute("create"); + if (create != null && !create.isEmpty()) { + createQueue = Boolean.parseBoolean(create); + } + } + LOG.debug("Specified rule instantiated with create flag: {}", + createQueue); + } + + /** + * Set the rule config just setting the create flag. + * @param create flag to allow queue creation for this rule + */ + public void setConfig(Boolean create) { + createQueue = create.booleanValue(); + LOG.debug("Specified rule instantiated with create flag: {}", + createQueue); + } + + @Override + public boolean initialize(ResourceScheduler scheduler) throws IOException { + if (!(scheduler instanceof FairScheduler)) { + throw new IOException( + "Specified rule can only be configured for the FairScheduler"); + } + if (getParentRule() != null) { + throw new IOException( + "Parent rule should not be configured for Specified rule."); + } + + FairScheduler fs = (FairScheduler) scheduler; + queueManager = fs.getQueueManager(); + + return true; + } + + @Override + public ApplicationPlacementContext getPlacementForApp( + ApplicationSubmissionContext asc, String user) throws YarnException { + + // Sanity check the provided queue + String queueName = asc.getQueue(); + if (!isValidQueueName(queueName)) { + LOG.error("Specified queue name not valid: '{}'", queueName); + throw new YarnException("Application submitted by user " + user + + "with illegal queue name '" + queueName + "'."); + } + // On submission the requested queue will be set to "default" if no queue + // is specified: just check the next rule in that case + if (queueName.equals(YarnConfiguration.DEFAULT_QUEUE_NAME)) { + return null; + } + queueName = assureRoot(queueName); + // If we can create the queue in the rule or the queue exists return it + if (createQueue || configuredQueue(queueName)) { + return new ApplicationPlacementContext(queueName); + } + return null; + } + + /** + * Check if the queue exists and is part of the configuration i.e. not + * a {@link FSQueue#isDynamic()} queue. + * @param queueName name of the queue to check + * @return true if the queue exists and is a "configured" queue + */ + private boolean configuredQueue(String queueName) { + FSQueue queue = queueManager.getQueue(queueName); + return (queue != null && !queue.isDynamic()); + } +} diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/placement/UserPlacementRule.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/placement/UserPlacementRule.java new file mode 100644 index 00000000000..f08279e6f71 --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/placement/UserPlacementRule.java @@ -0,0 +1,159 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.yarn.server.resourcemanager.placement; + +import org.apache.hadoop.classification.InterfaceAudience; +import org.apache.hadoop.classification.InterfaceStability; +import org.apache.hadoop.yarn.api.records.ApplicationSubmissionContext; +import org.apache.hadoop.yarn.exceptions.YarnException; +import org.apache.hadoop.yarn.server.resourcemanager.scheduler.ResourceScheduler; +import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.FSLeafQueue; +import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.FSQueue; +import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.FairScheduler; +import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.QueueManager; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.w3c.dom.Element; + +import java.io.IOException; + +import static org.apache.hadoop.yarn.server.resourcemanager.placement.FairQueuePlacementUtils.DOT; +import static org.apache.hadoop.yarn.server.resourcemanager.placement.FairQueuePlacementUtils.assureRoot; +import static org.apache.hadoop.yarn.server.resourcemanager.placement.FairQueuePlacementUtils.cleanName; + +/** + * Places apps in queues by username of the submitter. + */ +@InterfaceAudience.Private +@InterfaceStability.Unstable +public class UserPlacementRule extends PlacementRule { + private static final Logger LOG = + LoggerFactory.getLogger(UserPlacementRule.class); + + private QueueManager queueManager; + + /** + * Set the config based on the type of object passed in. + * @param initArg the config to be set + */ + @Override + public void setConfig(Object initArg) { + if (null == initArg) { + LOG.debug("Null object passed in: no config set"); + return; + } + if (initArg instanceof Element) { + LOG.debug("Setting config from XML"); + setConfig(Element.class.cast(initArg)); + } else { + if (initArg instanceof Boolean) { + LOG.debug("Setting config from Boolean"); + setConfig(Boolean.class.cast(initArg)); + } else { + LOG.info("Unknown object type passed in as config for rule {}: {}", + getName(), initArg.getClass()); + } + } + } + + /** + * Set the rule config from the xml config. + * @param conf An xml element from the {@link FairScheduler#conf} + */ + public void setConfig(Element conf) { + // No config can be set when no policy is defined and we use defaults + createQueue = true; + if (conf != null) { + String create = conf.getAttribute("create"); + if (create != null && !create.isEmpty()) { + createQueue = Boolean.parseBoolean(create); + } + } + LOG.debug("User rule instantiated with create flag: {}", + createQueue); + } + + /** + * Set the rule config just setting the create flag. + * @param create flag to allow queue creation for this rule + */ + public void setConfig(Boolean create) { + createQueue = create.booleanValue(); + LOG.debug("User rule instantiated with create flag: {}", + createQueue); + } + + @Override + public boolean initialize(ResourceScheduler scheduler) throws IOException { + if (!(scheduler instanceof FairScheduler)) { + throw new IOException( + "User rule can only be configured for the FairScheduler"); + } + if (getParentRule() != null && + getParentRule().getName().equals(getName())) { + throw new IOException("Parent rule should not be the same type as the " + + "child rule (User)"); + } + + FairScheduler fs = (FairScheduler) scheduler; + queueManager = fs.getQueueManager(); + + return true; + } + + @Override + public ApplicationPlacementContext getPlacementForApp( + ApplicationSubmissionContext asc, String user) throws YarnException { + String queueName; + + String cleanUser = cleanName(user); + PlacementRule parentRule = getParentRule(); + if (parentRule != null) { + LOG.debug("User rule: parent rule found: {}", parentRule.getName()); + ApplicationPlacementContext parent = + parentRule.getPlacementForApp(asc, user); + if (parent == null || + queueManager.getQueue(parent.getQueue()) instanceof FSLeafQueue) { + LOG.debug("User rule: parent rule failed"); + return null; + } + LOG.debug("User rule: parent rule result: {}", parent.getQueue()); + queueName = parent.getQueue() + DOT + cleanUser; + } else { + queueName = assureRoot(cleanUser); + } + + // If we can create the queue in the rule or the queue exists return it + if (createQueue || configuredQueue(queueName)) { + return new ApplicationPlacementContext(queueName); + } + return null; + } + + /** + * Check if the queue exists and is part of the configuration i.e. not + * a {@link FSQueue#isDynamic()} queue. + * @param queueName name of the queue to check + * @return true if the queue exists and is a "configured" queue + */ + private boolean configuredQueue(String queueName) { + FSQueue queue = queueManager.getQueue(queueName); + return (queue != null && !queue.isDynamic()); + } +} diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/placement/TestFairQueuePlacementUtils.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/placement/TestFairQueuePlacementUtils.java new file mode 100644 index 00000000000..4658e773e01 --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/placement/TestFairQueuePlacementUtils.java @@ -0,0 +1,124 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.yarn.server.resourcemanager.placement; + +import org.junit.Test; + +import static org.apache.hadoop.yarn.server.resourcemanager.placement.FairQueuePlacementUtils.DOT; +import static org.apache.hadoop.yarn.server.resourcemanager.placement.FairQueuePlacementUtils.DOT_REPLACEMENT; +import static org.apache.hadoop.yarn.server.resourcemanager.placement.FairQueuePlacementUtils.ROOT_QUEUE; +import static org.apache.hadoop.yarn.server.resourcemanager.placement.FairQueuePlacementUtils.assureRoot; +import static org.apache.hadoop.yarn.server.resourcemanager.placement.FairQueuePlacementUtils.cleanName; +import static org.apache.hadoop.yarn.server.resourcemanager.placement.FairQueuePlacementUtils.isValidQueueName; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertNotEquals; +import static org.junit.Assert.assertNull; +import static org.junit.Assert.assertTrue; + +/** + * Tests of the utility methods from {@link FairQueuePlacementUtils}. + */ +public class TestFairQueuePlacementUtils { + + /** + * Test name trimming and dot replacement in names. + */ + @Test + public void testCleanName() { + // permutations of dot placements + final String clean = "clean"; + final String dotted = "not.clean"; + final String multiDot = "more.un.clean"; + final String seqDot = "not..clean"; + final String unTrimmed = " .invalid. "; // not really a valid queue + + String cleaned = cleanName(clean); + assertEquals("Cleaned name was changed for clean input", clean, cleaned); + cleaned = cleanName(dotted); + assertFalse("Cleaned name contains dots", cleaned.contains(DOT)); + cleaned = cleanName(multiDot); + assertFalse("Cleaned name contains dots", cleaned.contains(DOT)); + assertNotEquals("Multi dot replacement failed", + cleaned.indexOf(DOT_REPLACEMENT), + cleaned.lastIndexOf(DOT_REPLACEMENT)); + cleaned = cleanName(seqDot); + assertFalse("Cleaned name contains dots", cleaned.contains(DOT)); + assertNotEquals("Sequential dot replacement failed", + cleaned.indexOf(DOT_REPLACEMENT), + cleaned.lastIndexOf(DOT_REPLACEMENT)); + cleaned = cleanName(unTrimmed); + assertTrue("Trimming start failed", cleaned.startsWith(DOT_REPLACEMENT)); + assertTrue("Trimming end failed", cleaned.endsWith(DOT_REPLACEMENT)); + } + + @Test + public void testAssureRoot() { + // permutations of rooted queue names + final String queueName = "base"; + final String rootNoDot = "rootbase"; + final String alreadyRoot = "root.base"; + + String rooted = assureRoot(queueName); + assertTrue("Queue does not have root prefix", + rooted.startsWith(ROOT_QUEUE + DOT)); + rooted = assureRoot(rootNoDot); + assertTrue("Queue does not have root prefix", + rooted.startsWith(ROOT_QUEUE + DOT)); + assertEquals("root replaced not added", 5, rooted.lastIndexOf(ROOT_QUEUE)); + rooted = assureRoot(alreadyRoot); + assertEquals("Root prefixed queue changed", rooted, alreadyRoot); + assertNull("Null queue should return null", + assureRoot(null)); + assertEquals("Empty queue should return empty", "", + assureRoot("")); + } + + @Test + public void testIsValidQueueName() { + // permutations of valid/invalid names + final String valid = "valid"; + final String validRooted = "root.valid"; + final String startDot = ".invalid"; + final String endDot = "invalid."; + final String startSpace = " invalid"; + final String endSpace = "invalid "; + final String unicodeSpace = "\u00A0invalid"; + + assertFalse("null queue is not valid", + isValidQueueName(null)); + assertTrue("empty queue should be valid", + isValidQueueName("")); + assertTrue("Should have been valid (valid)", + isValidQueueName(valid)); + assertTrue("Should have been valid (valid rooted)", + isValidQueueName(validRooted)); + assertFalse("Should have been invalid (star dot)", + isValidQueueName(startDot)); + assertFalse("Should have been invalid (end dot)", + isValidQueueName(endDot)); + assertFalse("Should have been invalid (start trim)", + isValidQueueName(startSpace)); + assertFalse("Should have been invalid (end trim)", + isValidQueueName(endSpace)); + // just one for sanity check extensive tests are in the scheduler utils + assertFalse("Should have been invalid (unicode trim)", + isValidQueueName(unicodeSpace)); + } +} \ No newline at end of file diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/placement/TestPlacementFactory.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/placement/TestPlacementFactory.java new file mode 100644 index 00000000000..f4d17330089 --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/placement/TestPlacementFactory.java @@ -0,0 +1,75 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.yarn.server.resourcemanager.placement; + +import org.junit.Test; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertNotNull; +import static org.junit.Assert.fail; + +/** + * Test for the {@link PlacementFactory}. + */ +public class TestPlacementFactory { + + /** + * Check that non existing class throws exception. + * + * @throws ClassNotFoundException + */ + @Test(expected = ClassNotFoundException.class) + public void testGetNonExistRuleText() throws ClassNotFoundException { + final String nonExist = "my.placement.Rule"; + PlacementFactory.getPlacementRule(nonExist, null); + } + + /** + * Check existing class using the class name. + * Relies on the {@link DefaultPlacementRule} of the FS. + */ + @Test + public void testGetExistRuleText() { + final String exists = DefaultPlacementRule.class.getCanonicalName(); + PlacementRule rule = null; + try { + rule = PlacementFactory.getPlacementRule(exists, null); + } catch (ClassNotFoundException cnfe) { + fail("Class should have been found"); + } + assertNotNull("Rule object is null", rule); + assertEquals("Names not equal", rule.getName(), exists); + } + + /** + * Existing class using the class reference. + * Relies on the {@link DefaultPlacementRule} of the FS. + */ + @Test + public void testGetRuleClass() { + PlacementRule rule = PlacementFactory.getPlacementRule( + DefaultPlacementRule.class, null); + assertNotNull("Rule object is null", rule); + // Should take anything as the second object: ignores unknown types in the + // default implementation. + rule = PlacementFactory.getPlacementRule( + DefaultPlacementRule.class, ""); + assertNotNull("Rule object is null", rule); + } +} diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/placement/TestPlacementRuleFS.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/placement/TestPlacementRuleFS.java new file mode 100644 index 00000000000..5ab01406234 --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/placement/TestPlacementRuleFS.java @@ -0,0 +1,115 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.yarn.server.resourcemanager.placement; + +import org.apache.hadoop.yarn.api.records.ApplicationSubmissionContext; +import org.apache.hadoop.yarn.server.resourcemanager.scheduler.ResourceScheduler; +import org.junit.Test; + +import static org.junit.Assert.assertNotNull; +import static org.junit.Assert.assertNull; +import static org.junit.Assert.assertTrue; + +/** + * Simple tests for FS specific parts of the PlacementRule + */ +public class TestPlacementRuleFS { + + /** + * Check setting the config and parent rules + */ + @Test + public void testRuleConfigs() { + PlacementRule rule = new TestRuleWithConfig(); + assertNotNull("Rule object is null", rule); + // Set the config similar to what the FS does. + rule.setConfig(Boolean.TRUE); + assertTrue("createQueue should be true", rule.createQueue); + // For the test rule it doesn't ignore object type and it throws. + boolean failed = false; + try { + rule.setConfig("false"); + } catch (IllegalArgumentException iae) { + // expected so ignore the exception: set the flag + failed = true; + } + assertTrue("Failed should have been set", failed); + } + + /** + * Default for setConfig is a noop, it should do nothing. + */ + @Test + public void testRuleNoConfig() { + PlacementRule rule = new TestRule(); + rule.setConfig(null); + } + + @Test + public void testRuleParent() { + PlacementRule rule = new TestRuleWithConfig(); + assertNotNull("Rule object is null", rule); + assertNull("Parent rule should not be set", rule.getParentRule()); + rule.setParentRule(new TestRule()); + assertNotNull("Parent rule should be set", rule.getParentRule()); + } + + /** + * Test placement rule. + */ + class TestRule extends PlacementRule { + @Override + public boolean initialize(ResourceScheduler resourceScheduler) { + return true; + } + + @Override + public ApplicationPlacementContext getPlacementForApp( + ApplicationSubmissionContext asc, String user) { + return null; + } + } + + /** + * Test placement rule that implements a setConfig like the FS rules. + * In this implementation the {@link #setConfig} throws a RTE to allow for + * testing. + */ + class TestRuleWithConfig extends PlacementRule { + @Override + public boolean initialize(ResourceScheduler resourceScheduler) { + return true; + } + + @Override + public ApplicationPlacementContext getPlacementForApp( + ApplicationSubmissionContext asc, String user) { + return null; + } + + @Override + public void setConfig(Object init) { + if (init instanceof Boolean) { + createQueue = Boolean.class.cast(init); + } else { + throw new IllegalArgumentException("Not a boolean"); + } + } + } +}