diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/MutableCSConfigurationProvider.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/MutableCSConfigurationProvider.java index 16632c5a1b..ea38098104 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/MutableCSConfigurationProvider.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/MutableCSConfigurationProvider.java @@ -38,6 +38,7 @@ import java.io.IOException; import java.util.ArrayList; import java.util.Arrays; +import java.util.Collection; import java.util.HashMap; import java.util.List; import java.util.Map; @@ -140,8 +141,27 @@ public void mutateConfiguration(UserGroupInformation user, updateQueue(updateQueueInfo, proposedConf, confUpdate); } for (Map.Entry global : mutationInfo.getGlobalParams() - .entrySet()) { + .getParams().entrySet()) { confUpdate.put(global.getKey(), global.getValue()); + if (global.getValue() == null) { + proposedConf.unset(global.getKey()); + } else { + proposedConf.set(global.getKey(), global.getValue()); + } + } + for (Map.Entry globalAdd : mutationInfo.getGlobalParams().getAddParams().entrySet()) { + Collection confCollection = proposedConf.getStringCollection(globalAdd.getKey()); + confCollection.add(globalAdd.getValue()); + String confVal = Joiner.on(",").join(confCollection); + confUpdate.put(globalAdd.getKey(), confVal); + proposedConf.set(globalAdd.getKey(), confVal); + } + for (Map.Entry globalRemove : mutationInfo.getGlobalParams().getRemoveParams().entrySet()) { + Collection confCollection = proposedConf.getStringCollection(globalRemove.getKey()); + confCollection.remove(globalRemove.getValue());//TODO this might be slow. + String confVal = Joiner.on(",").join(confCollection); + confUpdate.put(globalRemove.getKey(), confVal); + proposedConf.set(globalRemove.getKey(), confVal); } return confUpdate; } diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/QueueAdminConfigurationMutationACLPolicy.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/QueueAdminConfigurationMutationACLPolicy.java index 0a82d50cfe..dceb45a34f 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/QueueAdminConfigurationMutationACLPolicy.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/QueueAdminConfigurationMutationACLPolicy.java @@ -57,7 +57,7 @@ public void init(Configuration config, RMContext context) { public boolean isMutationAllowed(UserGroupInformation user, SchedConfUpdateInfo confUpdate) { // If there are global config changes, check if user is admin. - Map globalParams = confUpdate.getGlobalParams(); + Map globalParams = confUpdate.getGlobalParams().getParams(); if (globalParams != null && globalParams.size() != 0) { if (!authorizer.isAdmin(user)) { return false; diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/ConfigInfo.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/ConfigInfo.java new file mode 100644 index 0000000000..8a5d3cb0bd --- /dev/null +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/ConfigInfo.java @@ -0,0 +1,69 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.yarn.server.resourcemanager.webapp.dao; + +import java.util.HashMap; +import java.util.Map; + +import javax.xml.bind.annotation.XmlAccessType; +import javax.xml.bind.annotation.XmlAccessorType; +import javax.xml.bind.annotation.XmlElement; +import javax.xml.bind.annotation.XmlRootElement; +import javax.xml.bind.annotation.XmlTransient; + +/** + * Information for adding or updating a queue to scheduler configuration + * for this queue. + */ +@XmlTransient +public abstract class ConfigInfo { + + private HashMap params = new HashMap<>(); + + private HashMap addParams = new HashMap<>(); + + private HashMap removeParams = new HashMap<>(); + + public ConfigInfo() { } + + public ConfigInfo(Map params, Map addParams, Map removeParams) { + if (params != null) { + this.params = new HashMap<>(params); + } + if (addParams != null) { + this.addParams = new HashMap<>(addParams); + } + if (removeParams != null) { + this.removeParams = new HashMap<>(removeParams); + } + } + + public HashMap getParams() { + return this.params; + } + + public HashMap getAddParams() { + return this.addParams; + } + + public HashMap getRemoveParams() { + return this.removeParams; + } + +} \ No newline at end of file diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/GlobalConfigInfo.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/GlobalConfigInfo.java new file mode 100644 index 0000000000..5067acbb71 --- /dev/null +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/GlobalConfigInfo.java @@ -0,0 +1,41 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.yarn.server.resourcemanager.webapp.dao; + +import javax.xml.bind.annotation.XmlAccessType; +import javax.xml.bind.annotation.XmlAccessorType; +import javax.xml.bind.annotation.XmlRootElement; +import javax.xml.bind.annotation.XmlTransient; +import java.util.HashMap; +import java.util.Map; + +/** + * Information for adding or updating a queue to scheduler configuration + * for this queue. + */ +@XmlRootElement +@XmlAccessorType(XmlAccessType.FIELD) +public class GlobalConfigInfo extends ConfigInfo { + + public GlobalConfigInfo() { } + + public GlobalConfigInfo(Map params, Map addParams, Map removeParams) { + super(params, addParams, removeParams); + } +} \ No newline at end of file diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/QueueConfigInfo.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/QueueConfigInfo.java index b20eda6a13..2c9ee86ff0 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/QueueConfigInfo.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/QueueConfigInfo.java @@ -25,6 +25,7 @@ import javax.xml.bind.annotation.XmlAccessorType; import javax.xml.bind.annotation.XmlElement; import javax.xml.bind.annotation.XmlRootElement; +import javax.xml.bind.annotation.XmlTransient; /** * Information for adding or updating a queue to scheduler configuration @@ -32,26 +33,23 @@ */ @XmlRootElement @XmlAccessorType(XmlAccessType.FIELD) -public class QueueConfigInfo { +public class QueueConfigInfo extends ConfigInfo { @XmlElement(name = "queueName") private String queue; - private HashMap params = new HashMap<>(); - public QueueConfigInfo() { } + public QueueConfigInfo(String queue) { + this.queue = queue; + } + public QueueConfigInfo(String queue, Map params) { + super(params, null, null); this.queue = queue; - this.params = new HashMap<>(params); } public String getQueue() { return this.queue; } - - public HashMap getParams() { - return this.params; - } - } \ No newline at end of file diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/SchedConfUpdateInfo.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/SchedConfUpdateInfo.java index b7c585e733..04cfdef526 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/SchedConfUpdateInfo.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/SchedConfUpdateInfo.java @@ -33,7 +33,7 @@ */ @XmlRootElement(name = "schedConf") @XmlAccessorType(XmlAccessType.FIELD) -public class SchedConfUpdateInfo { +public class SchedConfUpdateInfo extends ConfigInfo { @XmlElement(name = "add-queue") private ArrayList addQueueInfo = new ArrayList<>(); @@ -44,7 +44,8 @@ @XmlElement(name = "update-queue") private ArrayList updateQueueInfo = new ArrayList<>(); - private HashMap global = new HashMap<>(); + @XmlElement(name = "global-updates") + private GlobalConfigInfo global = new GlobalConfigInfo(); public SchedConfUpdateInfo() { // JAXB needs this @@ -62,8 +63,7 @@ public SchedConfUpdateInfo() { return updateQueueInfo; } - @XmlElementWrapper(name = "global-updates") - public HashMap getGlobalParams() { + public GlobalConfigInfo getGlobalParams() { return global; } } diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/TestConfigurationMutationACLPolicies.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/TestConfigurationMutationACLPolicies.java index 0f5a3d82c5..931df78220 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/TestConfigurationMutationACLPolicies.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/TestConfigurationMutationACLPolicies.java @@ -165,7 +165,7 @@ public void testQueueAdminPolicyGlobal() { SchedConfUpdateInfo updateInfo = new SchedConfUpdateInfo(); assertTrue(policy.isMutationAllowed(GOOD_USER, updateInfo)); assertTrue(policy.isMutationAllowed(BAD_USER, updateInfo)); - updateInfo.getGlobalParams().put("globalKey", "globalValue"); + updateInfo.getGlobalParams().getParams().put("globalKey", "globalValue"); assertTrue(policy.isMutationAllowed(GOOD_USER, updateInfo)); assertFalse(policy.isMutationAllowed(BAD_USER, updateInfo)); } diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesConfigurationMutation.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesConfigurationMutation.java index 5fbe36fd62..bf2919f8ab 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesConfigurationMutation.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesConfigurationMutation.java @@ -461,7 +461,7 @@ public void testGlobalConfChange() throws Exception { // Set maximum-applications to 30000. SchedConfUpdateInfo updateInfo = new SchedConfUpdateInfo(); - updateInfo.getGlobalParams().put(CapacitySchedulerConfiguration.PREFIX + + updateInfo.getGlobalParams().getParams().put(CapacitySchedulerConfiguration.PREFIX + "maximum-applications", "30000"); response = @@ -476,7 +476,7 @@ public void testGlobalConfChange() throws Exception { ((CapacityScheduler) rm.getResourceScheduler()).getConfiguration(); assertEquals(30000, newCSConf.getMaximumSystemApplications()); - updateInfo.getGlobalParams().put(CapacitySchedulerConfiguration.PREFIX + + updateInfo.getGlobalParams().getParams().put(CapacitySchedulerConfiguration.PREFIX + "maximum-applications", null); // Unset maximum-applications. Should be set to default. response =