diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ClientRMService.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ClientRMService.java index 1f7a847..353bfeb 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ClientRMService.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ClientRMService.java @@ -89,6 +89,7 @@ import org.apache.hadoop.yarn.server.resourcemanager.rmnode.RMNode; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.SchedulerNodeReport; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.YarnScheduler; +import org.apache.hadoop.yarn.server.resourcemanager.security.QueueACLsManager; import org.apache.hadoop.yarn.server.resourcemanager.security.RMDelegationTokenSecretManager; import org.apache.hadoop.yarn.server.resourcemanager.security.authorize.RMPolicyProvider; import org.apache.hadoop.yarn.server.security.ApplicationACLsManager; @@ -119,15 +120,18 @@ private final ApplicationACLsManager applicationsACLsManager; + private final QueueACLsManager queueACLsManager; + public ClientRMService(RMContext rmContext, YarnScheduler scheduler, RMAppManager rmAppManager, ApplicationACLsManager applicationACLsManager, - RMDelegationTokenSecretManager rmDTSecretManager) { + RMDelegationTokenSecretManager rmDTSecretManager, QueueACLsManager queueACLsManager) { super(ClientRMService.class.getName()); this.scheduler = scheduler; this.rmContext = rmContext; this.rmAppManager = rmAppManager; this.applicationsACLsManager = applicationACLsManager; this.rmDTSecretManager = rmDTSecretManager; + this.queueACLsManager = queueACLsManager; } @Override @@ -193,7 +197,8 @@ public InetSocketAddress getBindAddress() { private boolean checkAccess(UserGroupInformation callerUGI, String owner, ApplicationAccessType operationPerformed, ApplicationId applicationId) { return applicationsACLsManager.checkAccess(callerUGI, operationPerformed, - owner, applicationId); + owner, applicationId) + && queueACLsManager.checkAccess(callerUGI, applicationId); } ApplicationId getNewApplicationId() { diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMAppManager.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMAppManager.java index 6439df1..4140cca 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMAppManager.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMAppManager.java @@ -51,6 +51,7 @@ import org.apache.hadoop.yarn.server.resourcemanager.rmapp.attempt.RMAppAttemptImpl; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.SchedulerUtils; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.YarnScheduler; +import org.apache.hadoop.yarn.server.resourcemanager.security.QueueACLsManager; import org.apache.hadoop.yarn.server.security.ApplicationACLsManager; import org.apache.hadoop.yarn.server.utils.BuilderUtils; @@ -70,16 +71,19 @@ private final ApplicationMasterService masterService; private final YarnScheduler scheduler; private final ApplicationACLsManager applicationACLsManager; + private final QueueACLsManager queueACLsManager; private Configuration conf; public RMAppManager(RMContext context, YarnScheduler scheduler, ApplicationMasterService masterService, - ApplicationACLsManager applicationACLsManager, Configuration conf) { + ApplicationACLsManager applicationACLsManager, Configuration conf, + QueueACLsManager queueACLsManager) { this.rmContext = context; this.scheduler = scheduler; this.masterService = masterService; this.applicationACLsManager = applicationACLsManager; this.conf = conf; + this.queueACLsManager = queueACLsManager; setCompletedAppsMax(conf.getInt( YarnConfiguration.RM_MAX_COMPLETED_APPLICATIONS, YarnConfiguration.DEFAULT_RM_MAX_COMPLETED_APPLICATIONS)); @@ -233,6 +237,7 @@ protected synchronized void checkAppNumCompletedLimit() { + " met. Removing app: " + removeId); rmContext.getRMApps().remove(removeId); this.applicationACLsManager.removeApplication(removeId); + this.queueACLsManager.removeApplication(removeId); } } diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ResourceManager.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ResourceManager.java index 841f387..52d57ef 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ResourceManager.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ResourceManager.java @@ -76,6 +76,7 @@ import org.apache.hadoop.yarn.server.resourcemanager.security.AMRMTokenSecretManager; import org.apache.hadoop.yarn.server.resourcemanager.security.ClientToAMTokenSecretManagerInRM; import org.apache.hadoop.yarn.server.resourcemanager.security.DelegationTokenRenewer; +import org.apache.hadoop.yarn.server.resourcemanager.security.QueueACLsManager; import org.apache.hadoop.yarn.server.resourcemanager.security.RMContainerTokenSecretManager; import org.apache.hadoop.yarn.server.resourcemanager.security.RMDelegationTokenSecretManager; import org.apache.hadoop.yarn.server.resourcemanager.security.NMTokenSecretManagerInRM; @@ -128,6 +129,7 @@ private EventHandler schedulerDispatcher; protected RMAppManager rmAppManager; protected ApplicationACLsManager applicationACLsManager; + protected QueueACLsManager queueACLsManager; protected RMDelegationTokenSecretManager rmDTSecretManager; private WebApp webApp; protected RMContext rmContext; @@ -209,8 +211,12 @@ protected void serviceInit(Configuration conf) throws Exception { this.nodesListManager); addService(nodesListManager); + this.queueACLsManager = new QueueACLsManager(); + // Initialize the scheduler this.scheduler = createScheduler(); + this.scheduler.registerQueueACLsManager(queueACLsManager); + System.out.println("I sm here"); this.schedulerDispatcher = createSchedulerEventDispatcher(); addIfService(this.schedulerDispatcher); this.rmDispatcher.register(SchedulerEventType.class, @@ -346,7 +352,7 @@ protected DelegationTokenRenewer createDelegationTokenRenewer() { protected RMAppManager createRMAppManager() { return new RMAppManager(this.rmContext, this.scheduler, this.masterService, - this.applicationACLsManager, this.conf); + this.applicationACLsManager, this.conf, this.queueACLsManager); } // sanity check for configurations @@ -710,7 +716,8 @@ protected ResourceTrackerService createResourceTrackerService() { protected ClientRMService createClientRMService() { return new ClientRMService(this.rmContext, scheduler, this.rmAppManager, - this.applicationACLsManager, this.rmDTSecretManager); + this.applicationACLsManager, this.rmDTSecretManager, + this.queueACLsManager); } protected ApplicationMasterService createApplicationMasterService() { @@ -791,6 +798,11 @@ public ApplicationACLsManager getApplicationACLsManager() { } @Private + public QueueACLsManager getQueueACLsManager() { + return this.queueACLsManager; + } + + @Private public RMContainerTokenSecretManager getRMContainerTokenSecretManager() { return this.containerTokenSecretManager; } diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/ResourceScheduler.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/ResourceScheduler.java index 8840881..2b30cf6 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/ResourceScheduler.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/ResourceScheduler.java @@ -25,6 +25,7 @@ import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.yarn.server.resourcemanager.RMContext; import org.apache.hadoop.yarn.server.resourcemanager.recovery.Recoverable; +import org.apache.hadoop.yarn.server.resourcemanager.security.QueueACLsManager; /** * This interface is the one implemented by the schedulers. It mainly extends @@ -40,4 +41,6 @@ * @throws IOException */ void reinitialize(Configuration conf, RMContext rmContext) throws IOException; + + void registerQueueACLsManager(QueueACLsManager queueACLsManager); } diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacityScheduler.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacityScheduler.java index 29c4d4b..17e7f7c 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacityScheduler.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacityScheduler.java @@ -75,6 +75,7 @@ import org.apache.hadoop.yarn.server.resourcemanager.scheduler.event.NodeRemovedSchedulerEvent; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.event.NodeUpdateSchedulerEvent; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.event.SchedulerEvent; +import org.apache.hadoop.yarn.server.resourcemanager.security.QueueACLsManager; import org.apache.hadoop.yarn.server.resourcemanager.security.RMContainerTokenSecretManager; import org.apache.hadoop.yarn.server.utils.Lock; import org.apache.hadoop.yarn.util.resource.ResourceCalculator; @@ -89,6 +90,8 @@ private static final Log LOG = LogFactory.getLog(CapacityScheduler.class); + private QueueACLsManager queueACLsManager; + private CSQueue root; private final static List EMPTY_CONTAINER_LIST = @@ -250,6 +253,7 @@ public Resource getClusterResources() { @Override public synchronized void reinitialize(Configuration conf, RMContext rmContext) throws IOException { + this.queueACLsManager.initial(); if (!initialized) { this.conf = new CapacitySchedulerConfiguration(conf); validateConf(this.conf); @@ -259,7 +263,7 @@ public Resource getClusterResources() { this.rmContext = rmContext; - initializeQueues(this.conf); + initializeQueues(this.conf, this.queueACLsManager); initialized = true; LOG.info("Initialized CapacityScheduler with " + @@ -273,7 +277,7 @@ public Resource getClusterResources() { validateConf(this.conf); try { LOG.info("Re-initializing queues..."); - reinitializeQueues(this.conf); + reinitializeQueues(this.conf, this.queueACLsManager); } catch (Throwable t) { this.conf = oldConf; throw new IOException("Failed to re-init queues", t); @@ -293,22 +297,24 @@ public CSQueue hook(CSQueue queue) { private static final QueueHook noop = new QueueHook(); @Lock(CapacityScheduler.class) - private void initializeQueues(CapacitySchedulerConfiguration conf) - throws IOException { - root = - parseQueue(this, conf, null, CapacitySchedulerConfiguration.ROOT, - queues, queues, noop); + private void initializeQueues(CapacitySchedulerConfiguration conf, + QueueACLsManager queueACLsManager) + throws IOException { + root = + parseQueue(this, conf, null, CapacitySchedulerConfiguration.ROOT, + queues, queues, noop, queueACLsManager); LOG.info("Initialized root queue " + root); } @Lock(CapacityScheduler.class) - private void reinitializeQueues(CapacitySchedulerConfiguration conf) + private void reinitializeQueues(CapacitySchedulerConfiguration conf, + QueueACLsManager queueACLsManager) throws IOException { // Parse new queues Map newQueues = new HashMap(); CSQueue newRoot = parseQueue(this, conf, null, CapacitySchedulerConfiguration.ROOT, - newQueues, queues, noop); + newQueues, queues, noop, queueACLsManager); // Ensure all existing queues are still present validateExistingQueues(queues, newQueues); @@ -361,7 +367,7 @@ static CSQueue parseQueue( CapacitySchedulerConfiguration conf, CSQueue parent, String queueName, Map queues, Map oldQueues, - QueueHook hook) throws IOException { + QueueHook hook, QueueACLsManager queueACLsManager) throws IOException { CSQueue queue; String[] childQueueNames = conf.getQueues((parent == null) ? @@ -373,7 +379,7 @@ static CSQueue parseQueue( } queue = new LeafQueue(csContext, queueName, parent,oldQueues.get(queueName)); - + ((LeafQueue)queue).registerQueueACLsManager(queueACLsManager); // Used only for unit tests queue = hook.hook(queue); } else { @@ -387,7 +393,7 @@ static CSQueue parseQueue( for (String childQueueName : childQueueNames) { CSQueue childQueue = parseQueue(csContext, conf, queue, childQueueName, - queues, oldQueues, hook); + queues, oldQueues, hook, queueACLsManager); childQueues.add(childQueue); } parentQueue.setChildQueues(childQueues); @@ -907,4 +913,8 @@ public void killContainer(RMContainer cont) { RMContainerEventType.KILL); } + @Override + public void registerQueueACLsManager(QueueACLsManager queueACLsManager) { + this.queueACLsManager = queueACLsManager; + } } diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/LeafQueue.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/LeafQueue.java index c2c5d27..d6ff027 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/LeafQueue.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/LeafQueue.java @@ -60,6 +60,7 @@ import org.apache.hadoop.yarn.server.resourcemanager.scheduler.common.fica.FiCaSchedulerApp; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.common.fica.FiCaSchedulerNode; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.common.fica.FiCaSchedulerUtils; +import org.apache.hadoop.yarn.server.resourcemanager.security.QueueACLsManager; import org.apache.hadoop.yarn.server.resourcemanager.security.RMContainerTokenSecretManager; import org.apache.hadoop.yarn.server.utils.BuilderUtils; import org.apache.hadoop.yarn.server.utils.Lock; @@ -127,6 +128,8 @@ private final ActiveUsersManager activeUsersManager; private final ResourceCalculator resourceCalculator; + + private QueueACLsManager queueACLsManager; public LeafQueue(CapacitySchedulerContext cs, String queueName, CSQueue parent, CSQueue old) { @@ -697,6 +700,7 @@ public void submitApplication(FiCaSchedulerApp application, String userName, LOG.info("Failed to submit application to parent-queue: " + getParent().getQueuePath(), ace); removeApplication(application, user); + this.queueACLsManager.removeApplication(application.getApplicationId()); throw ace; } } @@ -733,6 +737,8 @@ private synchronized void addApplication(FiCaSchedulerApp application, User user // Activate applications activateApplications(); + this.queueACLsManager.addApplication(application.getApplicationId(), this); + LOG.info("Application added -" + " appId: " + application.getApplicationId() + " user: " + user + "," + " leaf-queue: " + getQueueName() + @@ -774,7 +780,7 @@ public synchronized void removeApplication(FiCaSchedulerApp application, User us activeUsersManager.deactivateApplication( application.getUser(), application.getApplicationId()); } - + LOG.info("Application removed -" + " appId: " + application.getApplicationId() + " user: " + application.getUser() + @@ -1617,4 +1623,7 @@ public Resource getTotalResourcePending() { return ret; } + public void registerQueueACLsManager(QueueACLsManager queueACLsManager) { + this.queueACLsManager = queueACLsManager; + } } diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FairScheduler.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FairScheduler.java index 16b543c..e502b25 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FairScheduler.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FairScheduler.java @@ -81,6 +81,7 @@ import org.apache.hadoop.yarn.server.resourcemanager.scheduler.event.NodeRemovedSchedulerEvent; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.event.NodeUpdateSchedulerEvent; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.event.SchedulerEvent; +import org.apache.hadoop.yarn.server.resourcemanager.security.QueueACLsManager; import org.apache.hadoop.yarn.server.resourcemanager.security.RMContainerTokenSecretManager; import org.apache.hadoop.yarn.util.Clock; import org.apache.hadoop.yarn.util.SystemClock; @@ -1120,4 +1121,10 @@ public int getNumClusterNodes() { return nodes.size(); } + @Override + public void registerQueueACLsManager(QueueACLsManager queueACLsManager) { + // TODO DO NOTHING + + } + } diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fifo/FifoScheduler.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fifo/FifoScheduler.java index d971f3b..75c878a 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fifo/FifoScheduler.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fifo/FifoScheduler.java @@ -84,6 +84,7 @@ import org.apache.hadoop.yarn.server.resourcemanager.scheduler.event.NodeRemovedSchedulerEvent; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.event.NodeUpdateSchedulerEvent; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.event.SchedulerEvent; +import org.apache.hadoop.yarn.server.resourcemanager.security.QueueACLsManager; import org.apache.hadoop.yarn.server.utils.BuilderUtils; import org.apache.hadoop.yarn.server.utils.Lock; import org.apache.hadoop.yarn.util.resource.DefaultResourceCalculator; @@ -827,4 +828,9 @@ public QueueMetrics getRootQueueMetrics() { return DEFAULT_QUEUE.getMetrics(); } + @Override + public void registerQueueACLsManager(QueueACLsManager queueACLsManager) { + // DO NOTHING + } + } diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/security/QueueACLsManager.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/security/QueueACLsManager.java new file mode 100644 index 0000000..e1e37ad --- /dev/null +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/security/QueueACLsManager.java @@ -0,0 +1,69 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.yarn.server.resourcemanager.security; + +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.ConcurrentMap; + +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.apache.hadoop.classification.InterfaceAudience; +import org.apache.hadoop.security.UserGroupInformation; +import org.apache.hadoop.yarn.api.records.ApplicationId; +import org.apache.hadoop.yarn.api.records.QueueACL; +import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CSQueue; + +@InterfaceAudience.Private +public class QueueACLsManager { + private static final Log LOG = LogFactory + .getLog(QueueACLsManager.class); + + private final ConcurrentMap queueACLs = + new ConcurrentHashMap(); + + public void initial() { + queueACLs.clear(); + } + + public void addApplication(ApplicationId appId, CSQueue queue) { + this.queueACLs.put(appId, queue); + } + + public void removeApplication(ApplicationId appId) { + this.queueACLs.remove(appId); + } + + public boolean checkAccess(UserGroupInformation callerUGI, + ApplicationId appId) { + QueueACL acl = QueueACL.ADMINISTER_QUEUE; + + if (LOG.isDebugEnabled()) { + LOG.debug("Verifying queue access-type " + acl + " for " + + callerUGI + " for application " + appId); + } + CSQueue queue = this.queueACLs.get(appId); + + if(queue == null) { + LOG.debug("ACL not found for queue access-type " + acl + + " for application " + appId + " owned by " + + callerUGI); + return false; + } + return queue.hasAccess(acl, callerUGI); + } +} diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/AppBlock.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/AppBlock.java index edc5970..ce2f720 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/AppBlock.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/AppBlock.java @@ -38,6 +38,7 @@ import org.apache.hadoop.yarn.server.resourcemanager.RMContext; import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMApp; import org.apache.hadoop.yarn.server.resourcemanager.rmapp.attempt.RMAppAttempt; +import org.apache.hadoop.yarn.server.resourcemanager.security.QueueACLsManager; import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.AppAttemptInfo; import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.AppInfo; import org.apache.hadoop.yarn.server.resourcemanager.ResourceManager; @@ -53,11 +54,14 @@ public class AppBlock extends HtmlBlock { private ApplicationACLsManager aclsManager; + private QueueACLsManager queueACLsManager; @Inject - AppBlock(ResourceManager rm, ViewContext ctx, ApplicationACLsManager aclsManager) { + AppBlock(ResourceManager rm, ViewContext ctx, + ApplicationACLsManager aclsManager, QueueACLsManager queueACLsManager) { super(ctx); this.aclsManager = aclsManager; + this.queueACLsManager = queueACLsManager; } @Override @@ -91,8 +95,9 @@ protected void render(Block html) { callerUGI = UserGroupInformation.createRemoteUser(remoteUser); } if (callerUGI != null - && !this.aclsManager.checkAccess(callerUGI, - ApplicationAccessType.VIEW_APP, app.getUser(), appID)) { + && !(this.aclsManager.checkAccess(callerUGI, + ApplicationAccessType.VIEW_APP, app.getUser(), appID) && + this.queueACLsManager.checkAccess(callerUGI, appID))) { puts("You (User " + remoteUser + ") are not authorized to view application " + appID); return; diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMWebApp.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMWebApp.java index 90b0824..5a0980e 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMWebApp.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMWebApp.java @@ -22,6 +22,7 @@ import org.apache.hadoop.yarn.server.resourcemanager.RMContext; import org.apache.hadoop.yarn.server.resourcemanager.ResourceManager; +import org.apache.hadoop.yarn.server.resourcemanager.security.QueueACLsManager; import org.apache.hadoop.yarn.server.security.ApplicationACLsManager; import org.apache.hadoop.yarn.webapp.GenericExceptionHandler; import org.apache.hadoop.yarn.webapp.WebApp; @@ -48,6 +49,7 @@ public void setup() { bind(RMContext.class).toInstance(rm.getRMContext()); bind(ApplicationACLsManager.class).toInstance( rm.getApplicationACLsManager()); + bind(QueueACLsManager.class).toInstance(rm.getQueueACLsManager()); } route("/", RmController.class); route(pajoin("/nodes", NODE_STATE), RmController.class, "nodes"); diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMWebServices.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMWebServices.java index 12213a3..287d4f1 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMWebServices.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMWebServices.java @@ -53,6 +53,7 @@ import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CSQueue; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacityScheduler; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fifo.FifoScheduler; +import org.apache.hadoop.yarn.server.resourcemanager.security.QueueACLsManager; import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.AppAttemptInfo; import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.AppAttemptsInfo; import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.AppInfo; @@ -81,14 +82,17 @@ private static RecordFactory recordFactory = RecordFactoryProvider .getRecordFactory(null); private final ApplicationACLsManager aclsManager; + private final QueueACLsManager queueACLsManager; private @Context HttpServletResponse response; @Inject public RMWebServices(final ResourceManager rm, - final ApplicationACLsManager aclsManager) { + final ApplicationACLsManager aclsManager, + final QueueACLsManager queueACLsManager) { this.rm = rm; this.aclsManager = aclsManager; + this.queueACLsManager = queueACLsManager; } protected Boolean hasAccess(RMApp app, HttpServletRequest hsr) { @@ -99,9 +103,10 @@ protected Boolean hasAccess(RMApp app, HttpServletRequest hsr) { callerUGI = UserGroupInformation.createRemoteUser(remoteUser); } if (callerUGI != null - && !this.aclsManager.checkAccess(callerUGI, + && !(this.aclsManager.checkAccess(callerUGI, ApplicationAccessType.VIEW_APP, app.getUser(), - app.getApplicationId())) { + app.getApplicationId()) && this.queueACLsManager.checkAccess( + callerUGI, app.getApplicationId()))) { return false; } return true; diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/MockRM.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/MockRM.java index 522debb..1af0ca3 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/MockRM.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/MockRM.java @@ -59,6 +59,7 @@ import org.apache.hadoop.yarn.server.resourcemanager.rmnode.RMNodeEvent; import org.apache.hadoop.yarn.server.resourcemanager.rmnode.RMNodeEventType; import org.apache.hadoop.yarn.server.resourcemanager.rmnode.RMNodeImpl; +import org.apache.hadoop.yarn.server.resourcemanager.security.QueueACLsManager; import org.apache.hadoop.yarn.server.resourcemanager.security.RMDelegationTokenSecretManager; import org.apache.hadoop.yarn.util.Records; import org.apache.log4j.Level; @@ -290,7 +291,7 @@ public void sendAMLaunchFailed(ApplicationAttemptId appAttemptId) @Override protected ClientRMService createClientRMService() { return new ClientRMService(getRMContext(), getResourceScheduler(), - rmAppManager, applicationACLsManager, rmDTSecretManager) { + rmAppManager, applicationACLsManager, rmDTSecretManager, queueACLsManager) { @Override protected void serviceStart() { // override to not start rpc handler @@ -392,4 +393,7 @@ protected void startWepApp() { // override to disable webapp } + public QueueACLsManager getQueueACLsManager() { + return this.queueACLsManager; + } } diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestAppManager.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestAppManager.java index 6698412..daaf26f 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestAppManager.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestAppManager.java @@ -53,6 +53,7 @@ import org.apache.hadoop.yarn.server.resourcemanager.scheduler.ResourceScheduler; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.YarnScheduler; import org.apache.hadoop.yarn.server.resourcemanager.security.ClientToAMTokenSecretManagerInRM; +import org.apache.hadoop.yarn.server.resourcemanager.security.QueueACLsManager; import org.apache.hadoop.yarn.server.security.ApplicationACLsManager; import org.apache.hadoop.yarn.util.resource.Resources; import org.junit.After; @@ -141,15 +142,18 @@ public void handle(RMAppEvent event) { public class TestRMAppManager extends RMAppManager { public TestRMAppManager(RMContext context, Configuration conf) { - super(context, null, null, new ApplicationACLsManager(conf), conf); + super(context, null, null, new ApplicationACLsManager(conf), conf, + new QueueACLsManager()); setCompletedAppsMax(YarnConfiguration.DEFAULT_RM_MAX_COMPLETED_APPLICATIONS); } public TestRMAppManager(RMContext context, ClientToAMTokenSecretManagerInRM clientToAMSecretManager, YarnScheduler scheduler, ApplicationMasterService masterService, - ApplicationACLsManager applicationACLsManager, Configuration conf) { - super(context, scheduler, masterService, applicationACLsManager, conf); + ApplicationACLsManager applicationACLsManager, Configuration conf, + QueueACLsManager queueACLsManager) { + super(context, scheduler, masterService, applicationACLsManager, conf, + queueACLsManager); setCompletedAppsMax(YarnConfiguration.DEFAULT_RM_MAX_COMPLETED_APPLICATIONS); } @@ -202,7 +206,7 @@ public void setUp() { new ApplicationMasterService(rmContext, scheduler); appMonitor = new TestRMAppManager(rmContext, new ClientToAMTokenSecretManagerInRM(), scheduler, masterService, - new ApplicationACLsManager(conf), conf); + new ApplicationACLsManager(conf), conf, new QueueACLsManager()); appId = MockApps.newAppID(1); RecordFactory recordFactory = RecordFactoryProvider.getRecordFactory(null); @@ -409,7 +413,7 @@ public void testRMAppSubmitMaxAppAttempts() throws Exception { new ApplicationMasterService(rmContext, scheduler); TestRMAppManager appMonitor = new TestRMAppManager(rmContext, new ClientToAMTokenSecretManagerInRM(), scheduler, masterService, - new ApplicationACLsManager(conf), conf); + new ApplicationACLsManager(conf), conf, new QueueACLsManager()); ApplicationId appID = MockApps.newAppID(i * 4 + j + 1); asContext.setApplicationId(appID); diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestApplicationACLs.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestApplicationACLs.java index 8c28355..fd34062 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestApplicationACLs.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestApplicationACLs.java @@ -93,7 +93,7 @@ public static void setup() throws InterruptedException, IOException { resourceManager = new MockRM(conf) { protected ClientRMService createClientRMService() { return new ClientRMService(getRMContext(), this.scheduler, - this.rmAppManager, this.applicationACLsManager, null); + this.rmAppManager, this.applicationACLsManager, null, this.queueACLsManager); }; }; new Thread() { diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestClientRMService.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestClientRMService.java index 4817f45..46d3d52 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestClientRMService.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestClientRMService.java @@ -78,6 +78,7 @@ import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMAppEvent; import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMAppImpl; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.YarnScheduler; +import org.apache.hadoop.yarn.server.resourcemanager.security.QueueACLsManager; import org.apache.hadoop.yarn.server.resourcemanager.security.RMDelegationTokenSecretManager; import org.apache.hadoop.yarn.server.security.ApplicationACLsManager; import org.apache.hadoop.yarn.server.utils.BuilderUtils; @@ -119,7 +120,7 @@ public void testGetClusterNodes() throws Exception { protected ClientRMService createClientRMService() { return new ClientRMService(this.rmContext, scheduler, this.rmAppManager, this.applicationACLsManager, - this.rmDTSecretManager); + this.rmDTSecretManager, this.queueACLsManager); }; }; rm.start(); @@ -182,7 +183,7 @@ public void testGetApplicationReport() throws YarnException { when(rmContext.getRMApps()).thenReturn( new ConcurrentHashMap()); ClientRMService rmService = new ClientRMService(rmContext, null, null, - null, null); + null, null, null); RecordFactory recordFactory = RecordFactoryProvider.getRecordFactory(null); GetApplicationReportRequest request = recordFactory .newRecordInstance(GetApplicationReportRequest.class); @@ -203,7 +204,7 @@ public void testGetQueueInfo() throws Exception { RMContext rmContext = mock(RMContext.class); mockRMContext(yarnScheduler, rmContext); ClientRMService rmService = new ClientRMService(rmContext, yarnScheduler, - null, null, null); + null, null, null, null); GetQueueInfoRequest request = recordFactory .newRecordInstance(GetQueueInfoRequest.class); request.setQueueName("testqueue"); @@ -286,7 +287,7 @@ private void checkTokenRenewal(UserGroupInformation owner, RMContext rmContext = mock(RMContext.class); ClientRMService rmService = new ClientRMService( - rmContext, null, null, null, dtsm); + rmContext, null, null, null, dtsm, null); rmService.renewDelegationToken(request); } @@ -298,8 +299,10 @@ public void testAppSubmit() throws Exception { mockRMContext(yarnScheduler, rmContext); RMStateStore stateStore = mock(RMStateStore.class); when(rmContext.getStateStore()).thenReturn(stateStore); - RMAppManager appManager = new RMAppManager(rmContext, yarnScheduler, - null, mock(ApplicationACLsManager.class), new Configuration()); + RMAppManager appManager = + new RMAppManager(rmContext, yarnScheduler, + null, mock(ApplicationACLsManager.class), new Configuration(), + mock(QueueACLsManager.class)); when(rmContext.getDispatcher().getEventHandler()).thenReturn( new EventHandler() { public void handle(Event event) {} @@ -307,12 +310,16 @@ public void handle(Event event) {} ApplicationId appId1 = getApplicationId(100); ApplicationACLsManager mockAclsManager = mock(ApplicationACLsManager.class); + QueueACLsManager mockQueueACLsManager = mock(QueueACLsManager.class); when( mockAclsManager.checkAccess(UserGroupInformation.getCurrentUser(), ApplicationAccessType.VIEW_APP, null, appId1)).thenReturn(true); + when( + mockQueueACLsManager.checkAccess(UserGroupInformation.getCurrentUser(), + appId1)).thenReturn(true); ClientRMService rmService = new ClientRMService(rmContext, yarnScheduler, appManager, - mockAclsManager, null); + mockAclsManager, null, mockQueueACLsManager); // without name and queue @@ -386,8 +393,10 @@ public void testConcurrentAppSubmit() mockRMContext(yarnScheduler, rmContext); RMStateStore stateStore = mock(RMStateStore.class); when(rmContext.getStateStore()).thenReturn(stateStore); - RMAppManager appManager = new RMAppManager(rmContext, yarnScheduler, - null, mock(ApplicationACLsManager.class), new Configuration()); + RMAppManager appManager = + new RMAppManager(rmContext, yarnScheduler, + null, mock(ApplicationACLsManager.class), new Configuration(), + mock(QueueACLsManager.class)); final ApplicationId appId1 = getApplicationId(100); final ApplicationId appId2 = getApplicationId(101); @@ -422,7 +431,7 @@ public void handle(Event rawEvent) { when(rmContext.getDispatcher().getEventHandler()).thenReturn(eventHandler); final ClientRMService rmService = - new ClientRMService(rmContext, yarnScheduler, appManager, null, null); + new ClientRMService(rmContext, yarnScheduler, appManager, null, null, null); // submit an app and wait for it to block while in app submission Thread t = new Thread() { diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestClientRMTokens.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestClientRMTokens.java index 68caa9b..5a61d11 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestClientRMTokens.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestClientRMTokens.java @@ -63,6 +63,7 @@ import org.apache.hadoop.yarn.security.client.RMDelegationTokenIdentifier; import org.apache.hadoop.yarn.server.resourcemanager.recovery.NullRMStateStore; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.ResourceScheduler; +import org.apache.hadoop.yarn.server.resourcemanager.security.QueueACLsManager; import org.apache.hadoop.yarn.server.resourcemanager.security.RMDelegationTokenSecretManager; import org.apache.hadoop.yarn.server.security.ApplicationACLsManager; import org.apache.hadoop.yarn.server.utils.BuilderUtils; @@ -429,7 +430,7 @@ public ClientRMServiceForTest(Configuration conf, ResourceScheduler scheduler, RMDelegationTokenSecretManager rmDTSecretManager) { super(mock(RMContext.class), scheduler, mock(RMAppManager.class), - new ApplicationACLsManager(conf), rmDTSecretManager); + new ApplicationACLsManager(conf), rmDTSecretManager, new QueueACLsManager()); } // Use a random port unless explicitly specified. diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestApplicationLimits.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestApplicationLimits.java index f231685..cadeac6 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestApplicationLimits.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestApplicationLimits.java @@ -49,6 +49,7 @@ import org.apache.hadoop.yarn.server.resourcemanager.RMContext; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.common.fica.FiCaSchedulerApp; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.common.fica.FiCaSchedulerNode; +import org.apache.hadoop.yarn.server.resourcemanager.security.QueueACLsManager; import org.apache.hadoop.yarn.server.resourcemanager.security.RMContainerTokenSecretManager; import org.apache.hadoop.yarn.util.resource.DefaultResourceCalculator; import org.apache.hadoop.yarn.util.resource.ResourceCalculator; @@ -65,6 +66,7 @@ LeafQueue queue; private final ResourceCalculator resourceCalculator = new DefaultResourceCalculator(); + private final QueueACLsManager queueACLsManager = new QueueACLsManager(); @Before public void setUp() throws IOException { @@ -99,11 +101,12 @@ public void setUp() throws IOException { CSQueue root = CapacityScheduler.parseQueue(csContext, csConf, null, "root", queues, queues, - TestUtils.spyHook); + TestUtils.spyHook, queueACLsManager); queue = spy(new LeafQueue(csContext, A, root, null)); - + queueACLsManager.initial(); + queue.registerQueueACLsManager(queueACLsManager); // Stub out ACL checks doReturn(true). when(queue).hasAccess(any(QueueACL.class), @@ -170,7 +173,7 @@ public void testLimitsComputation() throws Exception { Map queues = new HashMap(); CSQueue root = CapacityScheduler.parseQueue(csContext, csConf, null, "root", - queues, queues, TestUtils.spyHook); + queues, queues, TestUtils.spyHook, queueACLsManager); LeafQueue queue = (LeafQueue)queues.get(A); @@ -259,7 +262,7 @@ public void testLimitsComputation() throws Exception { queues = new HashMap(); root = CapacityScheduler.parseQueue(csContext, csConf, null, "root", - queues, queues, TestUtils.spyHook); + queues, queues, TestUtils.spyHook, queueACLsManager); clusterResource = Resources.createResource(100 * 16 * GB); queue = (LeafQueue)queues.get(A); @@ -285,7 +288,7 @@ public void testLimitsComputation() throws Exception { queues = new HashMap(); root = CapacityScheduler.parseQueue(csContext, csConf, null, "root", - queues, queues, TestUtils.spyHook); + queues, queues, TestUtils.spyHook, queueACLsManager); queue = (LeafQueue)queues.get(A); assertEquals(9999, (int)csConf.getMaximumApplicationsPerQueue(queue.getQueuePath())); @@ -482,7 +485,7 @@ public void testHeadroom() throws Exception { Map queues = new HashMap(); CapacityScheduler.parseQueue(csContext, csConf, null, "root", - queues, queues, TestUtils.spyHook); + queues, queues, TestUtils.spyHook, queueACLsManager); // Manipulate queue 'a' LeafQueue queue = TestLeafQueue.stubLeafQueue((LeafQueue)queues.get(A)); diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacityScheduler.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacityScheduler.java index ec486d7..58a8458 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacityScheduler.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacityScheduler.java @@ -55,6 +55,7 @@ import org.apache.hadoop.yarn.server.resourcemanager.scheduler.event.NodeAddedSchedulerEvent; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.event.NodeRemovedSchedulerEvent; import org.apache.hadoop.yarn.server.resourcemanager.security.ClientToAMTokenSecretManagerInRM; +import org.apache.hadoop.yarn.server.resourcemanager.security.QueueACLsManager; import org.apache.hadoop.yarn.server.resourcemanager.security.RMContainerTokenSecretManager; import org.apache.hadoop.yarn.server.resourcemanager.security.NMTokenSecretManagerInRM; import org.apache.hadoop.yarn.util.resource.Resources; @@ -107,6 +108,8 @@ public void tearDown() throws Exception { @Test (timeout = 30000) public void testConfValidation() throws Exception { ResourceScheduler scheduler = new CapacityScheduler(); + QueueACLsManager queueACLsManager = new QueueACLsManager(); + scheduler.registerQueueACLsManager(queueACLsManager); Configuration conf = new YarnConfiguration(); conf.setInt(YarnConfiguration.RM_SCHEDULER_MINIMUM_ALLOCATION_MB, 2048); conf.setInt(YarnConfiguration.RM_SCHEDULER_MAXIMUM_ALLOCATION_MB, 1024); @@ -305,6 +308,8 @@ public void testMaximumCapacitySetup() { @Test public void testRefreshQueues() throws Exception { CapacityScheduler cs = new CapacityScheduler(); + QueueACLsManager queueACLsManager = new QueueACLsManager(); + cs.registerQueueACLsManager(queueACLsManager); CapacitySchedulerConfiguration conf = new CapacitySchedulerConfiguration(); setupQueueConfiguration(conf); cs.setConf(new YarnConfiguration()); @@ -399,6 +404,8 @@ private void checkNodeResourceUsage(int expected, @Test(expected=IOException.class) public void testParseQueue() throws IOException { CapacityScheduler cs = new CapacityScheduler(); + QueueACLsManager queueACLsManager = new QueueACLsManager(); + cs.registerQueueACLsManager(queueACLsManager); cs.setConf(new YarnConfiguration()); CapacitySchedulerConfiguration conf = new CapacitySchedulerConfiguration(); @@ -419,6 +426,8 @@ public void testReconnectedNode() throws Exception { new CapacitySchedulerConfiguration(); setupQueueConfiguration(csConf); CapacityScheduler cs = new CapacityScheduler(); + QueueACLsManager queueACLsManager = new QueueACLsManager(); + cs.registerQueueACLsManager(queueACLsManager); cs.setConf(new YarnConfiguration()); cs.reinitialize(csConf, new RMContextImpl(null, null, null, null, null, null, new RMContainerTokenSecretManager(csConf), @@ -444,6 +453,8 @@ public void testReconnectedNode() throws Exception { @Test public void testRefreshQueuesWithNewQueue() throws Exception { CapacityScheduler cs = new CapacityScheduler(); + QueueACLsManager queueACLsManager = new QueueACLsManager(); + cs.registerQueueACLsManager(queueACLsManager); CapacitySchedulerConfiguration conf = new CapacitySchedulerConfiguration(); setupQueueConfiguration(conf); cs.setConf(new YarnConfiguration()); diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestChildQueueOrder.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestChildQueueOrder.java index 014385c..7b4316d 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestChildQueueOrder.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestChildQueueOrder.java @@ -51,6 +51,7 @@ import org.apache.hadoop.yarn.server.resourcemanager.scheduler.NodeType; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.common.fica.FiCaSchedulerApp; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.common.fica.FiCaSchedulerNode; +import org.apache.hadoop.yarn.server.resourcemanager.security.QueueACLsManager; import org.apache.hadoop.yarn.server.utils.BuilderUtils; import org.apache.hadoop.yarn.util.resource.DefaultResourceCalculator; import org.apache.hadoop.yarn.util.resource.ResourceCalculator; @@ -215,7 +216,7 @@ public void testSortedQueues() throws Exception { CSQueue root = CapacityScheduler.parseQueue(csContext, csConf, null, CapacitySchedulerConfiguration.ROOT, queues, queues, - TestUtils.spyHook); + TestUtils.spyHook, new QueueACLsManager()); // Setup some nodes final int memoryPerNode = 10; diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestLeafQueue.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestLeafQueue.java index 40b73dc..a6958db 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestLeafQueue.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestLeafQueue.java @@ -31,6 +31,7 @@ import static org.mockito.Mockito.spy; import static org.mockito.Mockito.verify; import static org.mockito.Mockito.when; +import static org.mockito.Mockito.times; import java.io.IOException; import java.util.ArrayList; @@ -38,7 +39,6 @@ import java.util.HashMap; import java.util.List; import java.util.Map; - import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; import org.apache.hadoop.security.UserGroupInformation; @@ -64,6 +64,7 @@ import org.apache.hadoop.yarn.server.resourcemanager.scheduler.common.fica.FiCaSchedulerApp; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.common.fica.FiCaSchedulerNode; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.event.AppRemovedSchedulerEvent; +import org.apache.hadoop.yarn.server.resourcemanager.security.QueueACLsManager; import org.apache.hadoop.yarn.server.resourcemanager.security.RMContainerTokenSecretManager; import org.apache.hadoop.yarn.util.resource.DefaultResourceCalculator; import org.apache.hadoop.yarn.util.resource.ResourceCalculator; @@ -85,6 +86,7 @@ CapacityScheduler cs; CapacitySchedulerConfiguration csConf; CapacitySchedulerContext csContext; + QueueACLsManager spyQueueACLsManager; CSQueue root; Map queues = new HashMap(); @@ -93,10 +95,14 @@ final static String DEFAULT_RACK = "/default"; private final ResourceCalculator resourceCalculator = new DefaultResourceCalculator(); + private final QueueACLsManager queueACLsManager = new QueueACLsManager(); @Before public void setUp() throws Exception { + QueueACLsManager queueACLsManager = new QueueACLsManager(); + spyQueueACLsManager = spy(queueACLsManager); CapacityScheduler spyCs = new CapacityScheduler(); + spyCs.registerQueueACLsManager(spyQueueACLsManager); cs = spy(spyCs); rmContext = TestUtils.getMockRMContext(); @@ -133,7 +139,7 @@ public void setUp() throws Exception { CapacityScheduler.parseQueue(csContext, csConf, null, CapacitySchedulerConfiguration.ROOT, queues, queues, - TestUtils.spyHook); + TestUtils.spyHook, spyQueueACLsManager); cs.reinitialize(csConf, rmContext); } @@ -1626,7 +1632,7 @@ public void testActivateApplicationAfterQueueRefresh() throws Exception { CapacityScheduler.parseQueue(csContext, csConf, null, CapacitySchedulerConfiguration.ROOT, newQueues, queues, - TestUtils.spyHook); + TestUtils.spyHook, queueACLsManager); queues = newQueues; root.reinitialize(newRoot, cs.getClusterResources()); @@ -1651,7 +1657,7 @@ public void testNodeLocalityAfterQueueRefresh() throws Exception { CapacityScheduler.parseQueue(csContext, csConf, null, CapacitySchedulerConfiguration.ROOT, newQueues, queues, - TestUtils.spyHook); + TestUtils.spyHook, queueACLsManager); queues = newQueues; root.reinitialize(newRoot, cs.getClusterResources()); @@ -1991,6 +1997,35 @@ public void testMaxAMResourcePerQueuePercentAfterQueueRefresh() assertEquals(400, a.getMaximumActiveApplications()); } + @Test + public void testQueueACLsManager() throws Exception { + // Manipulate queue 'a' + LeafQueue a = stubLeafQueue((LeafQueue)queues.get(B)); + + // Users + final String user_0 = "user_0"; + + // Submit applications + final ApplicationAttemptId appAttemptId_0 = + TestUtils.getMockApplicationAttemptId(0, 0); + FiCaSchedulerApp app_0 = + new FiCaSchedulerApp(appAttemptId_0, user_0, a, + mock(ActiveUsersManager.class), rmContext); + a.submitApplication(app_0, user_0, B); + verify(spyQueueACLsManager, times(1)).addApplication( + app_0.getApplicationId(), a); + + final ApplicationAttemptId appAttemptId_1 = + TestUtils.getMockApplicationAttemptId(1, 0); + FiCaSchedulerApp app_1 = + new FiCaSchedulerApp(appAttemptId_1, user_0, a, + mock(ActiveUsersManager.class), rmContext); + a.submitApplication(app_1, user_0, B); // same user + verify(spyQueueACLsManager, times(1)).addApplication( + app_1.getApplicationId(), a); + + } + private CapacitySchedulerContext mockCSContext( CapacitySchedulerConfiguration csConf, Resource clusterResource) { CapacitySchedulerContext csContext = mock(CapacitySchedulerContext.class); diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestParentQueue.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestParentQueue.java index c5dbfde..2c4f578 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestParentQueue.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestParentQueue.java @@ -48,6 +48,7 @@ import org.apache.hadoop.yarn.server.resourcemanager.scheduler.NodeType; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.common.fica.FiCaSchedulerApp; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.common.fica.FiCaSchedulerNode; +import org.apache.hadoop.yarn.server.resourcemanager.security.QueueACLsManager; import org.apache.hadoop.yarn.util.resource.DefaultResourceCalculator; import org.apache.hadoop.yarn.util.resource.ResourceCalculator; import org.apache.hadoop.yarn.util.resource.Resources; @@ -203,7 +204,7 @@ public void testSingleLevelQueues() throws Exception { CSQueue root = CapacityScheduler.parseQueue(csContext, csConf, null, CapacitySchedulerConfiguration.ROOT, queues, queues, - TestUtils.spyHook); + TestUtils.spyHook, new QueueACLsManager()); // Setup some nodes final int memoryPerNode = 10; @@ -297,7 +298,7 @@ public void testSingleLevelQueuesPrecision() throws Exception { try { CapacityScheduler.parseQueue(csContext, csConf, null, CapacitySchedulerConfiguration.ROOT, queues, queues, - TestUtils.spyHook); + TestUtils.spyHook, new QueueACLsManager()); } catch (IllegalArgumentException ie) { exceptionOccured = true; } @@ -311,7 +312,7 @@ public void testSingleLevelQueuesPrecision() throws Exception { try { CapacityScheduler.parseQueue(csContext, csConf, null, CapacitySchedulerConfiguration.ROOT, queues, queues, - TestUtils.spyHook); + TestUtils.spyHook, new QueueACLsManager()); } catch (IllegalArgumentException ie) { exceptionOccured = true; } @@ -325,7 +326,7 @@ public void testSingleLevelQueuesPrecision() throws Exception { try { CapacityScheduler.parseQueue(csContext, csConf, null, CapacitySchedulerConfiguration.ROOT, queues, queues, - TestUtils.spyHook); + TestUtils.spyHook, new QueueACLsManager()); } catch (IllegalArgumentException ie) { exceptionOccured = true; } @@ -402,7 +403,7 @@ public void testMultiLevelQueues() throws Exception { CSQueue root = CapacityScheduler.parseQueue(csContext, csConf, null, CapacitySchedulerConfiguration.ROOT, queues, queues, - TestUtils.spyHook); + TestUtils.spyHook, new QueueACLsManager()); // Setup some nodes final int memoryPerNode = 10; @@ -518,7 +519,7 @@ public void testQueueCapacitySettingChildZero() throws Exception { Map queues = new HashMap(); CapacityScheduler.parseQueue(csContext, csConf, null, CapacitySchedulerConfiguration.ROOT, queues, queues, - TestUtils.spyHook); + TestUtils.spyHook, new QueueACLsManager()); } @Test (expected=IllegalArgumentException.class) @@ -535,7 +536,7 @@ public void testQueueCapacitySettingParentZero() throws Exception { Map queues = new HashMap(); CapacityScheduler.parseQueue(csContext, csConf, null, CapacitySchedulerConfiguration.ROOT, queues, queues, - TestUtils.spyHook); + TestUtils.spyHook, new QueueACLsManager()); } @Test @@ -557,7 +558,7 @@ public void testQueueCapacityZero() throws Exception { try { CapacityScheduler.parseQueue(csContext, csConf, null, CapacitySchedulerConfiguration.ROOT, queues, queues, - TestUtils.spyHook); + TestUtils.spyHook, new QueueACLsManager()); } catch (IllegalArgumentException e) { fail("Failed to create queues with 0 capacity: " + e); } @@ -573,7 +574,7 @@ public void testOffSwitchScheduling() throws Exception { CSQueue root = CapacityScheduler.parseQueue(csContext, csConf, null, CapacitySchedulerConfiguration.ROOT, queues, queues, - TestUtils.spyHook); + TestUtils.spyHook, new QueueACLsManager()); // Setup some nodes final int memoryPerNode = 10; @@ -639,7 +640,7 @@ public void testOffSwitchSchedulingMultiLevelQueues() throws Exception { CSQueue root = CapacityScheduler.parseQueue(csContext, csConf, null, CapacitySchedulerConfiguration.ROOT, queues, queues, - TestUtils.spyHook); + TestUtils.spyHook, new QueueACLsManager()); // Setup some nodes final int memoryPerNode = 10; @@ -723,7 +724,7 @@ public void testQueueAcl() throws Exception { CSQueue root = CapacityScheduler.parseQueue(csContext, csConf, null, CapacitySchedulerConfiguration.ROOT, queues, queues, - TestUtils.spyHook); + TestUtils.spyHook, new QueueACLsManager()); UserGroupInformation user = UserGroupInformation.getCurrentUser(); // Setup queue configs diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestQueueParsing.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestQueueParsing.java index c86d6b3..a299c8a 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestQueueParsing.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestQueueParsing.java @@ -25,6 +25,7 @@ import org.apache.hadoop.yarn.conf.YarnConfiguration; import org.apache.hadoop.yarn.server.resourcemanager.RMContextImpl; import org.apache.hadoop.yarn.server.resourcemanager.security.ClientToAMTokenSecretManagerInRM; +import org.apache.hadoop.yarn.server.resourcemanager.security.QueueACLsManager; import org.apache.hadoop.yarn.server.resourcemanager.security.RMContainerTokenSecretManager; import org.apache.hadoop.yarn.server.resourcemanager.security.NMTokenSecretManagerInRM; import org.junit.Test; @@ -43,6 +44,8 @@ public void testQueueParsing() throws Exception { YarnConfiguration conf = new YarnConfiguration(csConf); CapacityScheduler capacityScheduler = new CapacityScheduler(); + QueueACLsManager queueACLsManager = new QueueACLsManager(); + capacityScheduler.registerQueueACLsManager(queueACLsManager); capacityScheduler.setConf(conf); capacityScheduler.reinitialize(conf, new RMContextImpl(null, null, null, null, null, null, new RMContainerTokenSecretManager(conf), diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/security/TestClientToAMTokens.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/security/TestClientToAMTokens.java index fc2fda8..1d28ae2 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/security/TestClientToAMTokens.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/security/TestClientToAMTokens.java @@ -171,7 +171,7 @@ public void testClientToAMs() throws Exception { protected ClientRMService createClientRMService() { return new ClientRMService(this.rmContext, scheduler, this.rmAppManager, this.applicationACLsManager, - this.rmDTSecretManager); + this.rmDTSecretManager, this.queueACLsManager); }; @Override diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebApp.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebApp.java index aa2d6c6..07f4fa1 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebApp.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebApp.java @@ -46,6 +46,7 @@ import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacitySchedulerConfiguration; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fifo.FifoScheduler; import org.apache.hadoop.yarn.server.resourcemanager.security.ClientToAMTokenSecretManagerInRM; +import org.apache.hadoop.yarn.server.resourcemanager.security.QueueACLsManager; import org.apache.hadoop.yarn.server.resourcemanager.security.RMContainerTokenSecretManager; import org.apache.hadoop.yarn.server.resourcemanager.security.NMTokenSecretManagerInRM; import org.apache.hadoop.yarn.server.security.ApplicationACLsManager; @@ -200,6 +201,8 @@ public static CapacityScheduler mockCapacityScheduler() throws IOException { setupQueueConfiguration(conf); CapacityScheduler cs = new CapacityScheduler(); + QueueACLsManager queueACLsManager = new QueueACLsManager(); + cs.registerQueueACLsManager(queueACLsManager); cs.setConf(new YarnConfiguration()); cs.reinitialize(conf, new RMContextImpl(null, null, null, null, null, null, new RMContainerTokenSecretManager(conf),