diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/ACLsTestBase.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/ACLsTestBase.java index ddebaaa7d15..a011c8a8b35 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/ACLsTestBase.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/ACLsTestBase.java @@ -113,5 +113,9 @@ public ApplicationClientProtocol run() throws Exception { return userClient; } + public Configuration getConf() { + return conf; + } + protected abstract Configuration createConfiguration() throws IOException; } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/QueueACLsTestBase.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/QueueACLsTestBase.java index 82b3e24eaa2..a76d6298e96 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/QueueACLsTestBase.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/QueueACLsTestBase.java @@ -18,9 +18,12 @@ package org.apache.hadoop.yarn.server.resourcemanager; +import java.io.IOException; import java.util.HashMap; import java.util.Map; +import org.apache.hadoop.security.UserGroupInformation; +import org.apache.hadoop.yarn.api.records.QueueACL; import org.junit.Assert; import org.apache.hadoop.security.authorize.AccessControlList; @@ -43,6 +46,19 @@ public abstract class QueueACLsTestBase extends ACLsTestBase { + protected static final String QUEUE_C = "C"; + protected static final String QUEUE_C1 = "C1"; + private final String ALL_ACL = "*"; + private final String NONE_ACL = " "; + + + abstract public String getQUEUE_C(); + + abstract public String getQUEUE_C1(); + + abstract public void updateConfigWithCAndC1Queues(String rootACL, String queueCAcl, + String queueC1Acl) throws IOException; + @After public void tearDown() { if (resourceManager != null) { @@ -75,6 +91,69 @@ public void testApplicationACLs() throws Exception { } + @Test + public void testQueueAclRestrictedRootACL() throws IOException { + updateConfigWithCAndC1Queues(NONE_ACL, ALL_ACL, ALL_ACL); + checkAccess(false, true, true); + } + + @Test + public void testQueueAclNoAccess() throws IOException { + updateConfigWithCAndC1Queues(NONE_ACL, NONE_ACL, NONE_ACL); + checkAccess(false, false, false); + } + + @Test + public void testQueueAclRestrictedRootAndC1() throws IOException { + updateConfigWithCAndC1Queues(NONE_ACL, ALL_ACL, NONE_ACL); + checkAccess(false, true, true); + } + + @Test + public void testQueueAclRestrictedRootAndC() throws IOException { + updateConfigWithCAndC1Queues(NONE_ACL, NONE_ACL, ALL_ACL); + checkAccess(false, false, true); + } + + @Test + public void testQueueAclRestrictedC() throws IOException { + updateConfigWithCAndC1Queues(ALL_ACL, NONE_ACL, ALL_ACL); + checkAccess(true, true, true); + } + + @Test + public void testQueueAclRestrictedC1() throws IOException { + updateConfigWithCAndC1Queues(ALL_ACL, ALL_ACL, NONE_ACL); + checkAccess(true, true, true); + } + + @Test + public void testQueueAclDefaultValues() throws IOException { + updateConfigWithCAndC1Queues(null, null, null); + checkAccess(true, true, true); + } + + + private void checkAccess(boolean rootAccess, boolean cAccess, + boolean c1Access) throws IOException { + UserGroupInformation user = UserGroupInformation.getCurrentUser(); + + Assert.assertEquals(rootAccess, resourceManager.getResourceScheduler() + .checkAccess(user, QueueACL.ADMINISTER_QUEUE, "root")); + Assert.assertEquals(rootAccess, resourceManager.getResourceScheduler() + .checkAccess(user, QueueACL.SUBMIT_APPLICATIONS, "root")); + + Assert.assertEquals(cAccess, resourceManager.getResourceScheduler() + .checkAccess(user, QueueACL.ADMINISTER_QUEUE, getQUEUE_C())); + Assert.assertEquals(cAccess, resourceManager.getResourceScheduler() + .checkAccess(user, QueueACL.SUBMIT_APPLICATIONS, getQUEUE_C())); + + Assert.assertEquals(c1Access, resourceManager.getResourceScheduler() + .checkAccess(user, QueueACL.ADMINISTER_QUEUE, getQUEUE_C1())); + Assert.assertEquals(c1Access, resourceManager.getResourceScheduler() + .checkAccess(user, QueueACL.SUBMIT_APPLICATIONS, getQUEUE_C1())); + } + private void verifyGetClientAMToken(String submitter, String queueAdmin, String queueName, boolean setupACLs) throws Exception { ApplicationId applicationId = diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestAbsoluteResourceConfiguration.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestAbsoluteResourceConfiguration.java index fe4c880a56a..9b92b38e8c2 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestAbsoluteResourceConfiguration.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestAbsoluteResourceConfiguration.java @@ -165,10 +165,10 @@ public void testSimpleMinMaxResourceConfigurartionPerQueue() { Assert.assertEquals("Max resource configured for QUEUEB is not correct", QUEUE_B_MAXRES, csConf.getMaximumResourceRequirement("", QUEUEB_FULL, resourceTypes)); - Assert.assertEquals("Min resource configured for QUEUEC is not correct", + Assert.assertEquals("Min resource configured for QUEUE_C is not correct", QUEUE_C_MINRES, csConf.getMinimumResourceRequirement("", QUEUEC_FULL, resourceTypes)); - Assert.assertEquals("Max resource configured for QUEUEC is not correct", + Assert.assertEquals("Max resource configured for QUEUE_C is not correct", QUEUE_C_MAXRES, csConf.getMaximumResourceRequirement("", QUEUEC_FULL, resourceTypes)); } @@ -218,13 +218,13 @@ public void testEffectiveMinMaxResourceConfigurartionPerQueue() LeafQueue qC = (LeafQueue) cs.getQueue(QUEUEC); Assert.assertNotNull(qC); - Assert.assertEquals("Min resource configured for QUEUEC is not correct", + Assert.assertEquals("Min resource configured for QUEUE_C is not correct", QUEUE_C_MINRES, qC.queueResourceQuotas.getConfiguredMinResource()); - Assert.assertEquals("Max resource configured for QUEUEC is not correct", + Assert.assertEquals("Max resource configured for QUEUE_C is not correct", QUEUE_C_MAXRES, qC.queueResourceQuotas.getConfiguredMaxResource()); - Assert.assertEquals("Effective Min resource for QUEUEC is not correct", + Assert.assertEquals("Effective Min resource for QUEUE_C is not correct", QUEUE_C_MINRES, qC.queueResourceQuotas.getEffectiveMinResource()); - Assert.assertEquals("Effective Max resource for QUEUEC is not correct", + Assert.assertEquals("Effective Max resource for QUEUE_C is not correct", QUEUE_C_MAXRES, qC.queueResourceQuotas.getEffectiveMaxResource()); rm.stop(); @@ -311,13 +311,13 @@ public void testSimpleValidateAbsoluteResourceConfig() throws Exception { LeafQueue qC = (LeafQueue) cs.getQueue(QUEUEC); Assert.assertNotNull(qC); - Assert.assertEquals("Min resource configured for QUEUEC is not correct", + Assert.assertEquals("Min resource configured for QUEUE_C is not correct", QUEUE_C_MINRES, qC.queueResourceQuotas.getConfiguredMinResource()); - Assert.assertEquals("Max resource configured for QUEUEC is not correct", + Assert.assertEquals("Max resource configured for QUEUE_C is not correct", QUEUE_C_MAXRES, qC.queueResourceQuotas.getConfiguredMaxResource()); - Assert.assertEquals("Effective Min resource for QUEUEC is not correct", + Assert.assertEquals("Effective Min resource for QUEUE_C is not correct", QUEUE_C_MINRES, qC.queueResourceQuotas.getEffectiveMinResource()); - Assert.assertEquals("Effective Max resource for QUEUEC is not correct", + Assert.assertEquals("Effective Max resource for QUEUE_C is not correct", QUEUE_C_MAXRES, qC.queueResourceQuotas.getEffectiveMaxResource()); // 3. Create a new config and make sure one queue's min resource is more @@ -480,13 +480,13 @@ public void testEffectiveResourceAfterReducingClusterResource() LeafQueue qC = (LeafQueue) cs.getQueue(QUEUEC); Assert.assertNotNull(qC); - Assert.assertEquals("Min resource configured for QUEUEC is not correct", + Assert.assertEquals("Min resource configured for QUEUE_C is not correct", QUEUE_C_MINRES, qC.queueResourceQuotas.getConfiguredMinResource()); - Assert.assertEquals("Max resource configured for QUEUEC is not correct", + Assert.assertEquals("Max resource configured for QUEUE_C is not correct", QUEUE_C_MAXRES, qC.queueResourceQuotas.getConfiguredMaxResource()); - Assert.assertEquals("Effective Min resource for QUEUEC is not correct", + Assert.assertEquals("Effective Min resource for QUEUE_C is not correct", QUEUE_C_MINRES, qC.queueResourceQuotas.getEffectiveMinResource()); - Assert.assertEquals("Effective Max resource for QUEUEC is not correct", + Assert.assertEquals("Effective Max resource for QUEUE_C is not correct", QUEUE_C_MAXRES, qC.queueResourceQuotas.getEffectiveMaxResource()); // unregister one NM. @@ -505,9 +505,9 @@ public void testEffectiveResourceAfterReducingClusterResource() Assert.assertEquals("Effective Max resource for QUEUEB is not correct", QUEUEMAX_REDUCED, qB.queueResourceQuotas.getEffectiveMaxResource()); - Assert.assertEquals("Effective Min resource for QUEUEC is not correct", + Assert.assertEquals("Effective Min resource for QUEUE_C is not correct", QUEUEC_REDUCED, qC.queueResourceQuotas.getEffectiveMinResource()); - Assert.assertEquals("Effective Max resource for QUEUEC is not correct", + Assert.assertEquals("Effective Max resource for QUEUE_C is not correct", QUEUEMAX_REDUCED, qC.queueResourceQuotas.getEffectiveMaxResource()); rm.stop(); @@ -567,17 +567,17 @@ public void testEffectiveResourceAfterIncreasingClusterResource() LeafQueue qC = (LeafQueue) cs.getQueue(QUEUEC); Assert.assertNotNull(qC); - Assert.assertEquals("Min resource configured for QUEUEC is not correct", + Assert.assertEquals("Min resource configured for QUEUE_C is not correct", QUEUE_C_MINRES, qC.queueResourceQuotas.getConfiguredMinResource()); - Assert.assertEquals("Max resource configured for QUEUEC is not correct", + Assert.assertEquals("Max resource configured for QUEUE_C is not correct", QUEUE_C_MAXRES, qC.queueResourceQuotas.getConfiguredMaxResource()); - Assert.assertEquals("Effective Min resource for QUEUEC is not correct", + Assert.assertEquals("Effective Min resource for QUEUE_C is not correct", QUEUE_C_MINRES, qC.queueResourceQuotas.getEffectiveMinResource()); - Assert.assertEquals("Effective Max resource for QUEUEC is not correct", + Assert.assertEquals("Effective Max resource for QUEUE_C is not correct", QUEUE_C_MAXRES, qC.queueResourceQuotas.getEffectiveMaxResource()); - Assert.assertEquals("Absolute capacity for QUEUEC is not correct", + Assert.assertEquals("Absolute capacity for QUEUE_C is not correct", 0.2, qC.getAbsoluteCapacity(), DELTA); - Assert.assertEquals("Absolute Max capacity for QUEUEC is not correct", + Assert.assertEquals("Absolute Max capacity for QUEUE_C is not correct", 0.6, qC.getAbsoluteMaximumCapacity(), DELTA); LeafQueue qA1 = (LeafQueue) cs.getQueue(QUEUEA1); @@ -638,13 +638,13 @@ public void testEffectiveResourceAfterIncreasingClusterResource() Assert.assertEquals("Absolute Max capacity for QUEUEB is not correct", 0.4, qB.getAbsoluteMaximumCapacity(), DELTA); - Assert.assertEquals("Effective Min resource for QUEUEC is not correct", + Assert.assertEquals("Effective Min resource for QUEUE_C is not correct", QUEUE_C_MINRES, qC.queueResourceQuotas.getEffectiveMinResource()); - Assert.assertEquals("Effective Max resource for QUEUEC is not correct", + Assert.assertEquals("Effective Max resource for QUEUE_C is not correct", QUEUE_C_MAXRES, qC.queueResourceQuotas.getEffectiveMaxResource()); - Assert.assertEquals("Absolute capacity for QUEUEC is not correct", + Assert.assertEquals("Absolute capacity for QUEUE_C is not correct", 0.133, qC.getAbsoluteCapacity(), DELTA); - Assert.assertEquals("Absolute Max capacity for QUEUEC is not correct", + Assert.assertEquals("Absolute Max capacity for QUEUE_C is not correct", 0.4, qC.getAbsoluteMaximumCapacity(), DELTA); Assert.assertEquals("Effective Min resource for QUEUEB1 is not correct", diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacitySchedulerQueueACLs.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacitySchedulerQueueACLs.java index 5feb94b5879..2b6da951eac 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacitySchedulerQueueACLs.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacitySchedulerQueueACLs.java @@ -17,6 +17,7 @@ */ package org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity; +import java.io.IOException; import java.util.HashMap; import java.util.Map; @@ -71,4 +72,53 @@ protected Configuration createConfiguration() { return csConf; } + + @Override + public String getQUEUE_C() { + return QUEUE_C; + } + + @Override + public String getQUEUE_C1() { + return QUEUE_C1; + } + + @Override + public void updateConfigWithCAndC1Queues(String rootACL, String queueCAcl, + String queueC1Acl) throws IOException { + CapacitySchedulerConfiguration csConf = + (CapacitySchedulerConfiguration) getConf(); + csConf.clear(); + csConf.setQueues(CapacitySchedulerConfiguration.ROOT, + new String[] {QUEUE_C, QUEUEA, QUEUEB}); + + String cPath = CapacitySchedulerConfiguration.ROOT + "." + QUEUE_C; + String c1Path = cPath + "." + QUEUE_C1; + csConf.setQueues(cPath, new String[] {QUEUE_C1}); + csConf.setCapacity(c1Path, 100); + csConf.setCapacity(CapacitySchedulerConfiguration.ROOT + "." + + QUEUEA, 30f); + csConf.setCapacity(CapacitySchedulerConfiguration.ROOT + "." + + QUEUEB, 50f); + csConf.setCapacity(cPath, 20f); + + if (rootACL != null) { + csConf.setAcl(CapacitySchedulerConfiguration.ROOT, + QueueACL.SUBMIT_APPLICATIONS, rootACL); + csConf.setAcl(CapacitySchedulerConfiguration.ROOT, + QueueACL.ADMINISTER_QUEUE, rootACL); + } + + if(queueCAcl != null) { + csConf.setAcl(cPath, QueueACL.ADMINISTER_QUEUE, queueCAcl); + csConf.setAcl(cPath, QueueACL.SUBMIT_APPLICATIONS, queueCAcl); + } + + if(queueC1Acl != null) { + csConf.setAcl(c1Path, QueueACL.ADMINISTER_QUEUE, queueC1Acl); + csConf.setAcl(c1Path, QueueACL.SUBMIT_APPLICATIONS, queueC1Acl); + } + resourceManager.getResourceScheduler() + .reinitialize(csConf, resourceManager.getRMContext()); + } } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFairSchedulerQueueACLs.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFairSchedulerQueueACLs.java index ad56a209888..9f6213bf399 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFairSchedulerQueueACLs.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFairSchedulerQueueACLs.java @@ -18,6 +18,7 @@ package org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair; import java.io.File; +import java.io.IOException; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.yarn.conf.YarnConfiguration; @@ -57,4 +58,50 @@ protected Configuration createConfiguration() { return fsConf; } + + + + @Override + public String getQUEUE_C() { + return "root." + QUEUE_C; + } + + @Override + public String getQUEUE_C1() { + return "root."+ QUEUE_C + "." + QUEUE_C1; + } + + @Override + public void updateConfigWithCAndC1Queues(String rootACL, String queueCAcl, + String queueC1Acl) throws IOException { + FairSchedulerConfiguration fsConf = (FairSchedulerConfiguration) getConf(); + fsConf.clear(); + final String testDir = new File(System.getProperty("test.build.data", + "/tmp")).getAbsolutePath(); + final String allocFile = new File(testDir, "test-queues.xml") + .getAbsolutePath(); + + AllocationFileWriter.create() + .addQueue(new AllocationFileQueue.Builder("root") + .aclSubmitApps(rootACL) + .aclAdministerApps(rootACL) + .subQueue(new AllocationFileQueue.Builder(QUEUE_C) + .aclAdministerApps(queueCAcl) + .aclSubmitApps(queueCAcl) + .subQueue(new AllocationFileQueue.Builder(QUEUE_C1) + .aclSubmitApps(queueC1Acl) + .aclAdministerApps(queueC1Acl) + .build()) + .build()) + .build()) + .writeToFile(allocFile); + + fsConf.set(FairSchedulerConfiguration.ALLOCATION_FILE, allocFile); + + fsConf.setBoolean(YarnConfiguration.YARN_ACL_ENABLE, true); + fsConf.set(YarnConfiguration.RM_SCHEDULER, FairScheduler.class.getName()); + resourceManager.getResourceScheduler() + .reinitialize(fsConf, resourceManager.getRMContext()); + + } }