From 761dfaa6f4cb9023a227269b8d4eaef6fdb89461 Mon Sep 17 00:00:00 2001 From: Sunil Date: Sat, 5 Nov 2016 01:04:12 +0530 Subject: [PATCH] YARN-3955 --- .../hadoop/yarn/api/records/PriorityACL.java | 40 ++++++ .../apache/hadoop/yarn/security/AccessType.java | 1 + .../yarn/security/ConfiguredYarnAuthorizer.java | 4 +- .../hadoop/yarn/security/PrivilegedEntity.java | 3 +- .../yarn/server/resourcemanager/RMAppManager.java | 9 +- .../resourcemanager/scheduler/SchedulerUtils.java | 11 +- .../scheduler/capacity/AbstractCSQueue.java | 9 ++ .../scheduler/capacity/CSQueue.java | 8 ++ .../scheduler/capacity/CapacityScheduler.java | 24 ++++ .../capacity/CapacitySchedulerConfiguration.java | 143 ++++++++++++++++++++- .../scheduler/capacity/LeafQueue.java | 26 ++++ 11 files changed, 272 insertions(+), 6 deletions(-) create mode 100644 hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/PriorityACL.java diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/PriorityACL.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/PriorityACL.java new file mode 100644 index 0000000..05c4674 --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/PriorityACL.java @@ -0,0 +1,40 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.yarn.api.records; + +import org.apache.hadoop.classification.InterfaceAudience.Public; +import org.apache.hadoop.classification.InterfaceStability.Evolving; + +/** + * {@code PriorityACL} enumerates ACLs for application priority. + *

+ * The ACL is one of: + *

+ */ +@Public +@Evolving +public enum PriorityACL { + /** + * ACL to submit applications to the queue with given priority. + */ + SUBMIT_APP_PRIORITY, +} diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/security/AccessType.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/security/AccessType.java index 32459b9..3d97b75 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/security/AccessType.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/security/AccessType.java @@ -30,4 +30,5 @@ // queue SUBMIT_APP, ADMINISTER_QUEUE, + SUBMIT_APP_PRIORITY } \ No newline at end of file diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/security/ConfiguredYarnAuthorizer.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/security/ConfiguredYarnAuthorizer.java index 36c5214..f380298 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/security/ConfiguredYarnAuthorizer.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/security/ConfiguredYarnAuthorizer.java @@ -84,8 +84,8 @@ private boolean checkPermissionInternal(AccessType accessType, if (!queueName.contains(".")) { return ret; } - String parentQueueName = - queueName.substring(0, queueName.lastIndexOf(".")); + String parentQueueName = queueName.substring(0, + queueName.lastIndexOf(".")); return checkPermissionInternal(accessType, new PrivilegedEntity(target.getType(), parentQueueName), user); } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/security/PrivilegedEntity.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/security/PrivilegedEntity.java index 580bdf4..bfbbbac 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/security/PrivilegedEntity.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/security/PrivilegedEntity.java @@ -34,7 +34,8 @@ public class PrivilegedEntity { public enum EntityType { - QUEUE + QUEUE, + PRIORITY } EntityType type; diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMAppManager.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMAppManager.java index c065b60..bfe0f93 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMAppManager.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMAppManager.java @@ -35,6 +35,7 @@ import org.apache.hadoop.yarn.api.records.ApplicationId; import org.apache.hadoop.yarn.api.records.ApplicationSubmissionContext; import org.apache.hadoop.yarn.api.records.Priority; +import org.apache.hadoop.yarn.api.records.PriorityACL; import org.apache.hadoop.yarn.api.records.QueueACL; import org.apache.hadoop.yarn.api.records.ResourceRequest; import org.apache.hadoop.yarn.conf.YarnConfiguration; @@ -368,6 +369,7 @@ private RMAppImpl createAndPopulateNewRMApp( String queueName = submissionContext.getQueue(); String appName = submissionContext.getApplicationName(); CSQueue csqueue = ((CapacityScheduler) scheduler).getQueue(queueName); + Priority priority = submissionContext.getPriority(); if (null != csqueue && !authorizer.checkPermission( new AccessRequest(csqueue.getPrivilegedEntity(), userUgi, @@ -378,7 +380,12 @@ private RMAppImpl createAndPopulateNewRMApp( new AccessRequest(csqueue.getPrivilegedEntity(), userUgi, SchedulerUtils.toAccessType(QueueACL.ADMINISTER_QUEUE), applicationId.toString(), appName, Server.getRemoteAddress(), - null))) { + null)) + && !authorizer.checkPermission(new AccessRequest( + csqueue.getPriorityPrivilegedEntity(priority), userUgi, + SchedulerUtils.toAccessType(PriorityACL.SUBMIT_APP_PRIORITY), + applicationId.toString(), appName, Server.getRemoteAddress(), + null))) { throw RPCUtil.getRemoteException(new AccessControlException( "User " + user + " does not have permission to submit " + applicationId + " to queue " + submissionContext.getQueue())); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/SchedulerUtils.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/SchedulerUtils.java index c999e26..0b4663d 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/SchedulerUtils.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/SchedulerUtils.java @@ -29,6 +29,7 @@ import org.apache.hadoop.yarn.api.records.ContainerId; import org.apache.hadoop.yarn.api.records.ContainerState; import org.apache.hadoop.yarn.api.records.ContainerStatus; +import org.apache.hadoop.yarn.api.records.PriorityACL; import org.apache.hadoop.yarn.api.records.QueueACL; import org.apache.hadoop.yarn.api.records.QueueInfo; import org.apache.hadoop.yarn.api.records.Resource; @@ -379,7 +380,15 @@ public static AccessType toAccessType(QueueACL acl) { } return null; } - + + public static AccessType toAccessType(PriorityACL acl) { + switch (acl) { + case SUBMIT_APP_PRIORITY: + return AccessType.SUBMIT_APP_PRIORITY; + } + return null; + } + public static boolean checkResourceRequestMatchingNodePartition( String requestedPartition, String nodePartition, SchedulingMode schedulingMode) { diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/AbstractCSQueue.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/AbstractCSQueue.java index 096f5ea..6876c93 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/AbstractCSQueue.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/AbstractCSQueue.java @@ -24,6 +24,7 @@ import java.util.Iterator; import java.util.Map; import java.util.Set; +import java.util.TreeMap; import java.util.concurrent.locks.ReentrantReadWriteLock; import org.apache.commons.lang.StringUtils; @@ -98,6 +99,9 @@ protected ReentrantReadWriteLock.ReadLock readLock; protected ReentrantReadWriteLock.WriteLock writeLock; + // All priority ACLs entities have to be stored in a map. + protected Map priorityEntities = new TreeMap<>(); + public AbstractCSQueue(CapacitySchedulerContext cs, String queueName, CSQueue parent, CSQueue old) throws IOException { this.labelManager = cs.getRMContext().getNodeLabelManager(); @@ -195,6 +199,11 @@ public PrivilegedEntity getPrivilegedEntity() { } @Override + public PrivilegedEntity getPriorityPrivilegedEntity(Priority priority) { + return priorityEntities.get(priority); + } + + @Override public CSQueue getParent() { return parent; } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CSQueue.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CSQueue.java index daf7790..8f890d8 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CSQueue.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CSQueue.java @@ -30,6 +30,7 @@ import org.apache.hadoop.yarn.api.records.ApplicationAttemptId; import org.apache.hadoop.yarn.api.records.ApplicationId; import org.apache.hadoop.yarn.api.records.ContainerStatus; +import org.apache.hadoop.yarn.api.records.Priority; import org.apache.hadoop.yarn.api.records.QueueACL; import org.apache.hadoop.yarn.api.records.QueueState; import org.apache.hadoop.yarn.api.records.Resource; @@ -340,4 +341,11 @@ public void decreaseContainer(Resource clusterResource, * @return valid node labels */ public Set getNodeLabelsForQueue(); + + /** + * Get privileged entity object corresponding to priority. + * @param priority of the application + * @return privileged entity object corresponding to priority + */ + public PrivilegedEntity getPriorityPrivilegedEntity(Priority priority); } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacityScheduler.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacityScheduler.java index d759d47..95a300b 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacityScheduler.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacityScheduler.java @@ -45,6 +45,7 @@ import org.apache.hadoop.security.AccessControlException; import org.apache.hadoop.security.Groups; import org.apache.hadoop.security.UserGroupInformation; +import org.apache.hadoop.security.authorize.AccessControlList; import org.apache.hadoop.util.Time; import org.apache.hadoop.yarn.api.records.ApplicationAttemptId; import org.apache.hadoop.yarn.api.records.ApplicationId; @@ -66,6 +67,7 @@ import org.apache.hadoop.yarn.exceptions.YarnException; import org.apache.hadoop.yarn.exceptions.YarnRuntimeException; import org.apache.hadoop.yarn.proto.YarnServiceProtos.SchedulerResourceTypes; +import org.apache.hadoop.yarn.security.AccessType; import org.apache.hadoop.yarn.security.Permission; import org.apache.hadoop.yarn.security.YarnAuthorizationProvider; import org.apache.hadoop.yarn.server.resourcemanager.RMContext; @@ -529,6 +531,7 @@ private void initializeQueues(CapacitySchedulerConfiguration conf) LOG.info("Initialized root queue " + root); updatePlacementRules(); setQueueAcls(authorizer, queues); + setPriorityAcls(authorizer, queues); // Notify Preemption Manager preemptionManager.refreshQueues(null, root); @@ -560,6 +563,7 @@ private void reinitializeQueues(CapacitySchedulerConfiguration newConf) labelManager.reinitializeQueueLabels(getQueueToLabels()); setQueueAcls(authorizer, queues); + setPriorityAcls(authorizer, queues); // Notify Preemption Manager preemptionManager.refreshQueues(null, root); @@ -577,6 +581,26 @@ public static void setQueueAcls(YarnAuthorizationProvider authorizer, authorizer.setPermission(permissions, UserGroupInformation.getCurrentUser()); } + public static void setPriorityAcls(YarnAuthorizationProvider authorizer, + Map queues) throws IOException { + List permissions = new ArrayList<>(); + for (CSQueue queue : queues.values()) { + if (queue instanceof LeafQueue) { + LeafQueue lQueue = (LeafQueue) queue; + + for (Entry entry : lQueue.getPriorityACLs() + .entrySet()) { + Map acl = new HashMap<>(); + acl.put(AccessType.SUBMIT_APP_PRIORITY, entry.getValue()); + permissions.add( + new Permission(lQueue.getPriorityPrivilegedEntity(entry.getKey()), acl)); + } + } + } + authorizer.setPermission(permissions, + UserGroupInformation.getCurrentUser()); + } + private Map> getQueueToLabels() { Map> queueToLabels = new HashMap>(); for (CSQueue queue : queues.values()) { diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacitySchedulerConfiguration.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacitySchedulerConfiguration.java index cea5aa4..659af41 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacitySchedulerConfiguration.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacitySchedulerConfiguration.java @@ -27,6 +27,8 @@ import java.util.List; import java.util.Map; import java.util.Map.Entry; +import java.util.regex.Matcher; +import java.util.regex.Pattern; import java.util.Set; import java.util.StringTokenizer; @@ -38,6 +40,10 @@ import org.apache.hadoop.security.authorize.AccessControlList; import org.apache.hadoop.util.ReflectionUtils; import org.apache.hadoop.util.StringUtils; +import org.apache.hadoop.yarn.api.records.NodeId; +import org.apache.hadoop.yarn.api.records.NodeLabel; +import org.apache.hadoop.yarn.api.records.Priority; +import org.apache.hadoop.yarn.api.records.PriorityACL; import org.apache.hadoop.yarn.api.records.QueueACL; import org.apache.hadoop.yarn.api.records.QueueState; import org.apache.hadoop.yarn.api.records.ReservationACL; @@ -63,7 +69,13 @@ private static final Log LOG = LogFactory.getLog(CapacitySchedulerConfiguration.class); - + + public enum PriorityACLConfig { + USER, GROUP, MAX_PRIORITY, DEFAULT_PRIORITY + } + + public static final String PATTERN_FOR_PRIORITY_ACL = "\\[([^\\]]+)"; + private static final String CS_CONFIGURATION_FILE = "capacity-scheduler.xml"; @Private @@ -584,6 +596,10 @@ private static String getAclKey(ReservationACL acl) { return "acl_" + StringUtils.toLowerCase(acl.toString()); } + private static String getAclKey(PriorityACL acl) { + return "acl_" + StringUtils.toLowerCase(acl.toString()); + } + @Override public Map getReservationAcls(String queue) { @@ -632,6 +648,131 @@ public void setReservationAcls(String queue, } } + private HashMap getPrioirityAcl(String queue, + PriorityACL acl) { + String queuePrefix = getQueuePrefix(queue); + String defaultAcl = ALL_ACL; + String aclString = get(queuePrefix + getAclKey(acl), defaultAcl); + + Map> map = new HashMap<>(); + Matcher matcher = Pattern.compile(PATTERN_FOR_PRIORITY_ACL) + .matcher(aclString); + + /* + * Each ACL group will be separated by "[]". Syntax of each ACL set could be + * like below "max-priority=x user=y default-priority=z" + */ + while (matcher.find()) { + String str = matcher.group(1); + if (str.trim().isEmpty()) { + continue; + } + + // Assume max_priority will be coming as first entry in the acl string + Priority max_priority = null; + for (String part : str.trim().split(" +")) { + /* + * There are 3 possible options for key here: 1. user/group 2. + * max-priority 3. default-priority + * + * Categorize ACL map with max-priority as key and keep acl string as + * "user1,user2 group" format. + */ + String[] splits = part.split("="); + + max_priority = parsePriorityACLType(map, max_priority, splits); + } + } + + // Finally output map will have priority as key. Here value is acl string. + // Value could be a join of user and group. + HashMap output = new HashMap<>(); + for (Entry> entry : map.entrySet()) { + updateACLPerPriority(output, entry); + } + + return output; + } + + private void updateACLPerPriority(HashMap output, + Entry> entry) { + Priority priority = entry.getKey(); + List acls = entry.getValue(); + + String finalACL = new String(); + if (acls.get(0).equals(ALL_ACL) || acls.get(1).equals(ALL_ACL)) { + finalACL = ALL_ACL; + } else { + // skip last appended "," + String user = acls.get(0).substring(0, acls.get(0).length() - 2); + String group = acls.get(1).substring(0, acls.get(1).length() - 2); + finalACL = user + " " + group; + } + + output.put(priority, new AccessControlList(finalACL.trim())); + } + + private Priority parsePriorityACLType(Map> map, + Priority max_priority, String[] splits) { + // To easily parse all keys, convert to PriorityACLConfig enum. + PriorityACLConfig aclType = PriorityACLConfig + .valueOf(StringUtils.toUpperCase(splits[0].trim())); + switch (aclType) { + case MAX_PRIORITY : + max_priority = Priority.newInstance(Integer.parseInt(splits[1])); + break; + case USER : + createACLStringForPriority(map, max_priority, splits[1], + PriorityACLConfig.USER); + break; + case GROUP : + createACLStringForPriority(map, max_priority, splits[1], + PriorityACLConfig.GROUP); + break; + case DEFAULT_PRIORITY : + // ToDO + break; + } + return max_priority; + } + + /* + * This method will help to append user/group acl string against given + * priority. + */ + private void createACLStringForPriority( + Map> map, Priority max_priority, + String value, PriorityACLConfig type) { + // Fill ACL for each possible priority entries. + for (int i = 0; i <= max_priority.getPriority(); i++) { + List aclList = map.get(Priority.newInstance(i)); + if (aclList == null) { + aclList = new ArrayList<>(); + aclList.add(new StringBuilder()); + aclList.add(new StringBuilder()); + map.put(max_priority, aclList); + } + + StringBuilder str = aclList.get(type.ordinal() - 1); + if (str.equals(ALL_ACL)) { + return; + } + + if (value.trim().equals(ALL_ACL)) { + str.setLength(0); + str.append(value.trim()); + return; + } + + // Append a space as delim. + str.append(value).append(","); + } + } + + public Map getPriorityAcls(String queue) { + return getPrioirityAcl(queue, PriorityACL.SUBMIT_APP_PRIORITY); + } + public String[] getQueues(String queue) { LOG.debug("CSConf - getQueues called for: queuePrefix=" + getQueuePrefix(queue)); String[] queues = getStrings(getQueuePrefix(queue) + QUEUES); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/LeafQueue.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/LeafQueue.java index 214c6e7..fd9a6c3 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/LeafQueue.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/LeafQueue.java @@ -20,6 +20,7 @@ import java.io.IOException; import java.util.*; +import java.util.Map.Entry; import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.locks.ReentrantReadWriteLock; import java.util.concurrent.locks.ReentrantReadWriteLock.ReadLock; @@ -40,6 +41,7 @@ import org.apache.hadoop.yarn.api.records.ContainerExitStatus; import org.apache.hadoop.yarn.api.records.ContainerStatus; import org.apache.hadoop.yarn.api.records.Priority; +import org.apache.hadoop.yarn.api.records.PriorityACL; import org.apache.hadoop.yarn.api.records.QueueACL; import org.apache.hadoop.yarn.api.records.QueueInfo; import org.apache.hadoop.yarn.api.records.QueueState; @@ -50,6 +52,8 @@ import org.apache.hadoop.yarn.factory.providers.RecordFactoryProvider; import org.apache.hadoop.yarn.nodelabels.CommonNodeLabelsManager; import org.apache.hadoop.yarn.security.AccessType; +import org.apache.hadoop.yarn.security.PrivilegedEntity; +import org.apache.hadoop.yarn.security.PrivilegedEntity.EntityType; import org.apache.hadoop.yarn.server.resourcemanager.RMServerUtils; import org.apache.hadoop.yarn.server.resourcemanager.nodelabels.RMNodeLabelsManager; import org.apache.hadoop.yarn.server.resourcemanager.rmcontainer.RMContainer; @@ -134,6 +138,9 @@ private Map> ignorePartitionExclusivityRMContainers = new ConcurrentHashMap<>(); + Map priorityAcls = + new HashMap(); + @SuppressWarnings({ "unchecked", "rawtypes" }) public LeafQueue(CapacitySchedulerContext cs, String queueName, CSQueue parent, CSQueue old) throws IOException { @@ -194,6 +201,15 @@ protected void setupQueueConfigs(Resource clusterResource) conf.getMaximumApplicationMasterResourcePerQueuePercent( getQueuePath()); + priorityAcls = conf.getPriorityAcls(getQueuePath()); + for (Entry entry : priorityAcls.entrySet()) { + Priority priority = entry.getKey(); + StringBuilder str = new StringBuilder(); + str.append(priority.getPriority()); + priorityEntities.put(priority, + new PrivilegedEntity(EntityType.PRIORITY, str.toString())); + } + if (!SchedulerUtils.checkQueueLabelExpression(this.accessibleLabels, this.defaultLabelExpression, null)) { throw new IOException( @@ -486,6 +502,16 @@ private User getUserAndAddIfAbsent(String userName) { } } + @Private + public Map getPriorityACLs() { + try { + readLock.lock(); + return priorityAcls; + } finally { + readLock.unlock(); + } + } + @Override public void reinitialize( CSQueue newlyParsedQueue, Resource clusterResource) -- 2.7.4 (Apple Git-66)