diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java index 91728a3f5b..1a7c9f9086 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java @@ -608,6 +608,9 @@ public static boolean isAclEnabled(Configuration conf) { public static final String DEFAULT_CONFIGURATION_STORE = MEMORY_CONFIGURATION_STORE; + public static final String RM_SCHEDULER_MUTATION_POLICY_CLASS = + YARN_PREFIX + "scheduler.configuration.mutation.policy.class"; + public static final String YARN_AUTHORIZATION_PROVIDER = YARN_PREFIX + "authorization-provider"; private static final List RM_SERVICES_ADDRESS_CONF_KEYS_HTTP = diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/ConfigurationMutationPolicy.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/ConfigurationMutationPolicy.java new file mode 100644 index 0000000000..ef31cb2c09 --- /dev/null +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/ConfigurationMutationPolicy.java @@ -0,0 +1,47 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.yarn.server.resourcemanager.scheduler; + +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.security.UserGroupInformation; +import org.apache.hadoop.yarn.server.resourcemanager.RMContext; + +import java.util.Map; + +/** + * Interface for determining whether configuration mutations are allowed. + */ +public interface ConfigurationMutationPolicy { + + /** + * Initialize policy with configuration and RMContext. + * @param conf Configuration to initialize with. + * @param rmContext rmContext + */ + void init(Configuration conf, RMContext rmContext); + + /** + * Check if mutation is allowed. + * @param user User issuing the request + * @param confUpdate Key-value pairs for configurations to be updated. + */ + boolean isMutationAllowed(UserGroupInformation user, Map + confUpdate); + +} diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/ConfigurationMutationPolicyFactory.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/ConfigurationMutationPolicyFactory.java new file mode 100644 index 0000000000..5e43d6ec55 --- /dev/null +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/ConfigurationMutationPolicyFactory.java @@ -0,0 +1,48 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.yarn.server.resourcemanager.scheduler; + +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.util.ReflectionUtils; +import org.apache.hadoop.yarn.conf.YarnConfiguration; + +/** + * Factory class for creating instances of {@link ConfigurationMutationPolicy}. + */ +public final class ConfigurationMutationPolicyFactory { + + private static final Log LOG = LogFactory.getLog( + ConfigurationMutationPolicyFactory.class); + + private ConfigurationMutationPolicyFactory() { + // Unused. + } + + public static ConfigurationMutationPolicy getPolicy(Configuration conf) { + Class policyClass = + conf.getClass(YarnConfiguration.RM_SCHEDULER_MUTATION_POLICY_CLASS, + DefaultConfigurationMutationPolicy.class, + ConfigurationMutationPolicy.class); + LOG.info("Using ConfigurationMutationPolicy implementation - " + + policyClass); + return ReflectionUtils.newInstance(policyClass, conf); + } +} \ No newline at end of file diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/DefaultConfigurationMutationPolicy.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/DefaultConfigurationMutationPolicy.java new file mode 100644 index 0000000000..0893c5a2ee --- /dev/null +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/DefaultConfigurationMutationPolicy.java @@ -0,0 +1,47 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.yarn.server.resourcemanager.scheduler; + +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.security.UserGroupInformation; +import org.apache.hadoop.yarn.security.YarnAuthorizationProvider; +import org.apache.hadoop.yarn.server.resourcemanager.RMContext; +import org.apache.hadoop.yarn.server.resourcemanager.scheduler.ConfigurationMutationPolicy; + +import java.util.Map; + +/** + * Default configuration mutation policy. Checks if user is YARN admin. + */ +public class DefaultConfigurationMutationPolicy implements + ConfigurationMutationPolicy { + + private YarnAuthorizationProvider authorizer; + + @Override + public void init(Configuration conf, RMContext rmContext) { + authorizer = YarnAuthorizationProvider.getInstance(conf); + } + + @Override + public boolean isMutationAllowed(UserGroupInformation user, + Map confUpdate) { + return authorizer.isAdmin(user); + } +} diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/MutableConfigurationProvider.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/MutableConfigurationProvider.java index 889c3bc1f0..29541fc0f4 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/MutableConfigurationProvider.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/MutableConfigurationProvider.java @@ -18,6 +18,8 @@ package org.apache.hadoop.yarn.server.resourcemanager.scheduler; +import org.apache.hadoop.security.UserGroupInformation; + import java.io.IOException; import java.util.Map; @@ -32,7 +34,7 @@ * @param confUpdate Key-value pairs for configurations to be updated. * @throws IOException if scheduler could not be reinitialized */ - void mutateConfiguration(String user, Map confUpdate) - throws IOException; + void mutateConfiguration(UserGroupInformation user, Map + confUpdate) throws IOException; } diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacityScheduler.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacityScheduler.java index e027f5e79d..67a145222f 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacityScheduler.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacityScheduler.java @@ -2494,7 +2494,7 @@ public void updateConfiguration(UserGroupInformation user, Map confUpdate) throws IOException { if (csConfProvider instanceof MutableConfigurationProvider) { ((MutableConfigurationProvider) csConfProvider).mutateConfiguration( - user.getShortUserName(), confUpdate); + user, confUpdate); } else { throw new UnsupportedOperationException("Configured CS configuration " + "provider does not support updating configuration."); diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/MutableCSConfigurationProvider.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/MutableCSConfigurationProvider.java index ea1b3c070f..19e90eee67 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/MutableCSConfigurationProvider.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/MutableCSConfigurationProvider.java @@ -19,8 +19,12 @@ package org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.conf; import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.security.AccessControlException; +import org.apache.hadoop.security.UserGroupInformation; import org.apache.hadoop.yarn.conf.YarnConfiguration; import org.apache.hadoop.yarn.server.resourcemanager.RMContext; +import org.apache.hadoop.yarn.server.resourcemanager.scheduler.ConfigurationMutationPolicy; +import org.apache.hadoop.yarn.server.resourcemanager.scheduler.ConfigurationMutationPolicyFactory; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.MutableConfigurationProvider; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacitySchedulerConfiguration; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.conf.YarnConfigurationStore.LogMutation; @@ -38,6 +42,7 @@ private Configuration schedConf; private YarnConfigurationStore confStore; + private ConfigurationMutationPolicy mutationPolicy; private RMContext rmContext; private Configuration conf; @@ -68,6 +73,8 @@ public void init(Configuration config) throws IOException { schedConf.set(kv.getKey(), kv.getValue()); } confStore.initialize(config, schedConf); + this.mutationPolicy = ConfigurationMutationPolicyFactory.getPolicy(config); + mutationPolicy.init(config, rmContext); this.conf = config; } @@ -80,10 +87,14 @@ public CapacitySchedulerConfiguration loadConfiguration(Configuration } @Override - public void mutateConfiguration(String user, + public void mutateConfiguration(UserGroupInformation user, Map confUpdate) throws IOException { + if (!mutationPolicy.isMutationAllowed(user, confUpdate)) { + throw new AccessControlException("User is not admin of all modified" + + " queues."); + } Configuration oldConf = new Configuration(schedConf); - LogMutation log = new LogMutation(confUpdate, user); + LogMutation log = new LogMutation(confUpdate, user.getShortUserName()); long id = confStore.logMutation(log); for (Map.Entry kv : confUpdate.entrySet()) { if (kv.getValue() == null) { diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/QueueAdminConfigurationMutationPolicy.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/QueueAdminConfigurationMutationPolicy.java new file mode 100644 index 0000000000..dedb0bbf3d --- /dev/null +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/QueueAdminConfigurationMutationPolicy.java @@ -0,0 +1,103 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.conf; + +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.security.UserGroupInformation; +import org.apache.hadoop.yarn.api.records.QueueACL; +import org.apache.hadoop.yarn.server.resourcemanager.RMContext; +import org.apache.hadoop.yarn.server.resourcemanager.scheduler.ConfigurationMutationPolicy; +import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CSQueue; +import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacityScheduler; +import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacitySchedulerConfiguration; + +import java.util.Arrays; +import java.util.Collections; +import java.util.HashSet; +import java.util.List; +import java.util.Map; +import java.util.Set; +import java.util.regex.Matcher; +import java.util.regex.Pattern; + +/** + * A configuration mutation policy which checks that user has admin privileges + * on all queues they are changing. + */ +public class QueueAdminConfigurationMutationPolicy implements + ConfigurationMutationPolicy { + + private RMContext rmContext; + private static final List multipleSuffixConfigs; + + static { + multipleSuffixConfigs = Collections.unmodifiableList(Arrays.asList( + CapacitySchedulerConfiguration.ACCESSIBLE_NODE_LABELS, + CapacitySchedulerConfiguration.ORDERING_POLICY + )); + } + + @Override + public void init(Configuration conf, RMContext rmContext) { + this.rmContext = rmContext; + } + + @Override + public boolean isMutationAllowed(UserGroupInformation user, + Map confUpdate) { + Set queues = new HashSet<>(); + Pattern queuePattern = Pattern.compile(CapacitySchedulerConfiguration + .PREFIX + "(" + CapacitySchedulerConfiguration.ROOT + + "(\\.[^\\.]+)*)" + "\\.[^\\.]+"); + for (Map.Entry kv : confUpdate.entrySet()) { + Matcher m = queuePattern.matcher(kv.getKey()); + if (m.matches()) { + String queuePath = m.group(1); + for (String multipleSuffixConfig : multipleSuffixConfigs) { + int endQueueIndex = queuePath.indexOf(multipleSuffixConfig); + if (endQueueIndex != -1) { + queuePath = queuePath.substring(0, endQueueIndex - 1); + break; + } + } + queues.add(queuePath); + } + } + for (String queuePath : queues) { + String queueName = queuePath.lastIndexOf('.') != -1 ? + queuePath.substring(queuePath.lastIndexOf('.') + 1) : queuePath; + CSQueue queue = ((CapacityScheduler) rmContext.getScheduler()) + .getQueue(queueName); + String parentPath = queuePath; + while (queue == null) { + // We are adding a queue (whose parent we are possibly also adding). + // Check ACL of lowest parent queue which already exists. + parentPath = parentPath.substring(0, parentPath.lastIndexOf('.')); + String parentName = parentPath.lastIndexOf('.') != -1 ? + parentPath.substring(parentPath.lastIndexOf('.') + 1) : parentPath; + queue = ((CapacityScheduler) rmContext.getScheduler()) + .getQueue(parentName); + } + if (!queue.hasAccess(QueueACL.ADMINISTER_QUEUE, user)) { + return false; + } + } + return true; + } +} diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/TestMutableCSConfigurationProvider.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/TestMutableCSConfigurationProvider.java index 254da31893..02559b6a7c 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/TestMutableCSConfigurationProvider.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/TestMutableCSConfigurationProvider.java @@ -19,6 +19,7 @@ package org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.conf; import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.security.UserGroupInformation; import org.apache.hadoop.yarn.server.resourcemanager.RMContext; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacityScheduler; import org.junit.Before; @@ -47,7 +48,8 @@ private Map badUpdate; private CapacityScheduler cs; - private static final String TEST_USER = "testUser"; + private static final UserGroupInformation TEST_USER = UserGroupInformation + .createUserForTesting("testUser", new String[] {}); @Before public void setUp() {