Index: hadoop-common.yandex/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/QueuePlacementRule.java =================================================================== --- hadoop-common.yandex.orig/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/QueuePlacementRule.java +++ hadoop-common.yandex/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/QueuePlacementRule.java @@ -22,6 +22,7 @@ import java.util.Collection; import java.util.HashMap; import java.util.List; import java.util.Map; +import java.util.regex.Pattern; import org.apache.hadoop.security.Groups; import org.apache.hadoop.yarn.conf.YarnConfiguration; @@ -31,6 +32,7 @@ import org.w3c.dom.Node; public abstract class QueuePlacementRule { protected boolean create; + protected String prefix; /** * Initializes the rule with any arguments. @@ -40,6 +42,16 @@ public abstract class QueuePlacementRule */ public QueuePlacementRule initialize(boolean create, Map args) { this.create = create; + String parent = (args != null) ? args.get("parent") : null; + if (parent != null) { + if (parent.endsWith(".")) + parent = parent.substring(0, parent.length() - 1); + if (parent.startsWith("root")) + parent = parent.replace("root", ""); + this.prefix = "root." + parent + "."; + } else { + this.prefix = "root."; + } return this; } @@ -113,7 +125,7 @@ public abstract class QueuePlacementRule @Override protected String getQueueForApp(String requestedQueue, String user, Groups groups, Collection configuredQueues) { - return "root." + user; + return prefix + user; } @Override @@ -121,7 +133,73 @@ public abstract class QueuePlacementRule return create; } } - + + /** + * Places apps in queues by username of the submitter if username matched by pattern + */ + public static class UserMatch extends QueuePlacementRule { + + private Pattern pattern; + + @Override + public QueuePlacementRule initialize(boolean create, Map args) { + super.initialize(create, args); + String patternArg = args == null ? ".*" : args.get("pattern"); + if (patternArg == null) + patternArg = ".*"; + pattern = Pattern.compile(patternArg); + return this; + } + + @Override + protected String getQueueForApp(String requestedQueue, + String user, Groups groups, Collection configuredQueues) { + if (pattern.matcher(user).matches()) + return prefix + user; + else + return ""; + } + + @Override + public boolean isTerminal() { + return false; + } + } + + /** + * Places apps in queues by username of the submitter + * if primary group matched by pattern + */ + public static class PrimaryGroupMatch extends QueuePlacementRule { + + private Pattern pattern; + + @Override + public QueuePlacementRule initialize(boolean create, Map args) { + super.initialize(create, args); + String patternArg = args == null ? ".*" : args.get("pattern"); + if (patternArg == null) + patternArg = ".*"; + pattern = Pattern.compile(patternArg); + return this; + } + + @Override + protected String getQueueForApp(String requestedQueue, + String user, Groups groups, Collection configuredQueues) + throws IOException { + if (pattern.matcher(groups.getGroups(user).get(0)).matches()) + return prefix + user; + else + return ""; + } + + @Override + public boolean isTerminal() { + return false; + } + } + /** * Places apps in queues by primary group of the submitter */ @@ -130,7 +208,7 @@ public abstract class QueuePlacementRule protected String getQueueForApp(String requestedQueue, String user, Groups groups, Collection configuredQueues) throws IOException { - return "root." + groups.getGroups(user).get(0); + return prefix + groups.getGroups(user).get(0); } @Override @@ -138,7 +216,43 @@ public abstract class QueuePlacementRule return create; } } - + + /** + * Places apps in queues by username of the submitter if one of secondary + * groups matched by pattern + */ + public static class SecondaryGroupMatch extends QueuePlacementRule { + + private Pattern pattern; + + @Override + public QueuePlacementRule initialize(boolean create, Map args) { + super.initialize(create, args); + String patternArg = args == null ? ".*" : args.get("pattern"); + if (patternArg == null) + patternArg = ".*"; + pattern = Pattern.compile(patternArg); + return this; + } + + @Override + protected String getQueueForApp(String requestedQueue, + String user, Groups groups, Collection configuredQueues) + throws IOException { + List groupNames = groups.getGroups(user); + for (int i = 1; i < groupNames.size(); i++) { + if (pattern.matcher(groupNames.get(i)).matches()) + return prefix + user; + } + return ""; + } + + @Override + public boolean isTerminal() { + return false; + } + } + /** * Places apps in queues by secondary group of the submitter * @@ -176,8 +290,8 @@ public abstract class QueuePlacementRule if (requestedQueue.equals(YarnConfiguration.DEFAULT_QUEUE_NAME)) { return ""; } else { - if (!requestedQueue.startsWith("root.")) { - requestedQueue = "root." + requestedQueue; + if (!requestedQueue.startsWith(prefix)) { + requestedQueue = prefix + requestedQueue; } return requestedQueue; } Index: hadoop-common.yandex/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestQueuePlacementPolicy.java =================================================================== --- hadoop-common.yandex.orig/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestQueuePlacementPolicy.java +++ hadoop-common.yandex/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestQueuePlacementPolicy.java @@ -57,7 +57,27 @@ public class TestQueuePlacementPolicy { assertEquals("root.someuser", policy.assignAppToQueue("default", "someuser")); assertEquals("root.otheruser", policy.assignAppToQueue("default", "otheruser")); } - + + @Test + public void testSpecifiedUserPolicyWithPrefix() throws Exception { + StringBuffer sb = new StringBuffer(); + sb.append(""); + sb.append(" "); + sb.append(" "); + sb.append(" "); + sb.append(" "); + sb.append(" "); + sb.append(""); + QueuePlacementPolicy policy = parse(sb.toString()); + assertEquals("root.granted.specifiedq",policy.assignAppToQueue("specifiedq", "someuser")); + assertEquals("root.admin.admin1", policy.assignAppToQueue("default", "admin1")); + assertEquals("root.admin.primg.admin2", policy.assignAppToQueue("default", "admin2")); + assertEquals("root.admin.secg.admin3", policy.assignAppToQueue("default", "admin3")); + assertEquals("root.guests.someuser", policy.assignAppToQueue("default", "someuser")); + assertEquals("root.guests.otheruser", policy.assignAppToQueue("default", "otheruser")); + } + + @Test public void testNoCreate() throws Exception { StringBuffer sb = new StringBuffer(); Index: hadoop-common.yandex/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/QueuePlacementPolicy.java =================================================================== --- hadoop-common.yandex.orig/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/QueuePlacementPolicy.java +++ hadoop-common.yandex/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/QueuePlacementPolicy.java @@ -38,7 +38,10 @@ public class QueuePlacementPolicy { Map> map = new HashMap>(); map.put("user", QueuePlacementRule.User.class); + map.put("userMatch", QueuePlacementRule.UserMatch.class); map.put("primaryGroup", QueuePlacementRule.PrimaryGroup.class); + map.put("primaryGroupMatch", QueuePlacementRule.PrimaryGroupMatch.class); + map.put("secondaryGroupMatch", QueuePlacementRule.SecondaryGroupMatch.class); map.put("secondaryGroupExistingQueue", QueuePlacementRule.SecondaryGroupExistingQueue.class); map.put("specified", QueuePlacementRule.Specified.class);