diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java index ca71d35718e..204e9e8f63c 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java @@ -210,6 +210,12 @@ private static void addDeprecatedKeys() { public static final boolean DEFAULT_RM_SCHEDULER_USE_PORT_FOR_NODE_NAME = false; + /** Configured scheduler queue placement rules. */ + public static final String QUEUE_PLACEMENT_RULES = YARN_PREFIX + + "scheduler.queue-placement-rules"; + /** UserGroupMappingPlacementRule configuration string. */ + public static final String USER_GROUP_PLACEMENT_RULE = "user-group"; + /** Enable Resource Manager webapp ui actions */ public static final String RM_WEBAPP_UI_ACTIONS_ENABLED = RM_PREFIX + "webapp.ui-actions.enabled"; diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/placement/PlacementFactory.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/placement/PlacementFactory.java new file mode 100644 index 00000000000..dc5bae9bfae --- /dev/null +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/placement/PlacementFactory.java @@ -0,0 +1,41 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.yarn.server.resourcemanager.placement; + +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.util.ReflectionUtils; + +/** + * Factory class for creating instances of {@link PlacementRule}. + */ +public class PlacementFactory { + + private static final Log LOG = LogFactory.getLog(PlacementFactory.class); + + public static PlacementRule getPlacementRule(String ruleStr, + Configuration conf) + throws ClassNotFoundException { + Class ruleClass = Class.forName(ruleStr) + .asSubclass(PlacementRule.class); + LOG.info("Using PlacementRule implementation - " + ruleClass); + return ReflectionUtils.newInstance(ruleClass, conf); + } +} \ No newline at end of file diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacityScheduler.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacityScheduler.java index d3186da5b4c..114030489e8 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacityScheduler.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacityScheduler.java @@ -65,6 +65,7 @@ import org.apache.hadoop.yarn.proto.YarnServiceProtos.SchedulerResourceTypes; import org.apache.hadoop.yarn.server.resourcemanager.RMContext; import org.apache.hadoop.yarn.server.resourcemanager.nodelabels.RMNodeLabelsManager; +import org.apache.hadoop.yarn.server.resourcemanager.placement.PlacementFactory; import org.apache.hadoop.yarn.server.resourcemanager.placement.PlacementRule; import org.apache.hadoop.yarn.server.resourcemanager.placement.UserGroupMappingPlacementRule; import org.apache.hadoop.yarn.server.resourcemanager.placement.UserGroupMappingPlacementRule.QueueMapping; @@ -564,15 +565,37 @@ public int getPendingBacklogs() { } private void updatePlacementRules() throws IOException { + // Initialize placement rules + Collection placementRuleStrs = conf.getStringCollection( + YarnConfiguration.QUEUE_PLACEMENT_RULES); List placementRules = new ArrayList<>(); - - // Initialize UserGroupMappingPlacementRule - // TODO, need make this defineable by configuration. - UserGroupMappingPlacementRule ugRule = getUserGroupMappingPlacementRule(); - if (null != ugRule) { - placementRules.add(ugRule); + if (placementRuleStrs.isEmpty()) { + PlacementRule ugRule = getUserGroupMappingPlacementRule(); + if (null != ugRule) { + placementRules.add(ugRule); + } + } else { + for (String placementRuleStr : placementRuleStrs) { + switch (placementRuleStr) { + case YarnConfiguration.USER_GROUP_PLACEMENT_RULE: + PlacementRule ugRule = getUserGroupMappingPlacementRule(); + if (null != ugRule) { + placementRules.add(ugRule); + } + break; + default: + try { + PlacementRule rule = PlacementFactory.getPlacementRule( + placementRuleStr, conf); + if (null != rule) { + placementRules.add(rule); + } + } catch (ClassNotFoundException cnfe) { + throw new IOException(cnfe); + } + } + } } - rmContext.getQueuePlacementManager().updateRules(placementRules); }