diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java index fce78c9..06d6ab9 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java @@ -632,6 +632,9 @@ public static boolean isAclEnabled(Configuration conf) { public static final String DEFAULT_RM_CONFIGURATION_PROVIDER_CLASS = "org.apache.hadoop.yarn.LocalConfigurationProvider"; + public static final String RM_CONF_MUTATION_POLICY = RM_PREFIX + + "configuration.mutation.policy"; + public static final String YARN_AUTHORIZATION_PROVIDER = YARN_PREFIX + "authorization-provider"; private static final List RM_SERVICES_ADDRESS_CONF_KEYS_HTTP = diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMContext.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMContext.java index c9d185f..cad4a13 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMContext.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMContext.java @@ -28,6 +28,7 @@ import org.apache.hadoop.yarn.conf.ConfigurationProvider; import org.apache.hadoop.yarn.event.Dispatcher; import org.apache.hadoop.yarn.server.resourcemanager.ahs.RMApplicationHistoryWriter; +import org.apache.hadoop.yarn.server.resourcemanager.conf.MutableConfigurationManager; import org.apache.hadoop.yarn.server.resourcemanager.metrics.SystemMetricsPublisher; import org.apache.hadoop.yarn.server.resourcemanager.nodelabels.RMNodeLabelsManager; import org.apache.hadoop.yarn.server.resourcemanager.nodelabels.RMDelegatedNodeLabelsUpdater; @@ -40,7 +41,6 @@ import org.apache.hadoop.yarn.server.resourcemanager.rmcontainer.ContainerAllocationExpirer; import org.apache.hadoop.yarn.server.resourcemanager.rmnode.RMNode; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.ResourceScheduler; - import org.apache.hadoop.yarn.server.resourcemanager.scheduler.distributed.QueueLimitCalculator; import org.apache.hadoop.yarn.server.resourcemanager.security.AMRMTokenSecretManager; import org.apache.hadoop.yarn.server.resourcemanager.security.ClientToAMTokenSecretManagerInRM; @@ -154,4 +154,8 @@ void setRMDelegatedNodeLabelsUpdater( void setRMAppLifetimeMonitor(RMAppLifetimeMonitor rmAppLifetimeMonitor); RMAppLifetimeMonitor getRMAppLifetimeMonitor(); + + MutableConfigurationManager getMutableConfigurationManager(); + + void setMutableConfigurationManager(MutableConfigurationManager mutableConfMgr); } diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMContextImpl.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMContextImpl.java index dc8f7d1..9d53160 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMContextImpl.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMContextImpl.java @@ -32,6 +32,7 @@ import org.apache.hadoop.yarn.conf.ConfigurationProvider; import org.apache.hadoop.yarn.event.Dispatcher; import org.apache.hadoop.yarn.server.resourcemanager.ahs.RMApplicationHistoryWriter; +import org.apache.hadoop.yarn.server.resourcemanager.conf.MutableConfigurationManager; import org.apache.hadoop.yarn.server.resourcemanager.metrics.SystemMetricsPublisher; import org.apache.hadoop.yarn.server.resourcemanager.nodelabels.RMNodeLabelsManager; import org.apache.hadoop.yarn.server.resourcemanager.nodelabels.RMDelegatedNodeLabelsUpdater; @@ -44,7 +45,6 @@ import org.apache.hadoop.yarn.server.resourcemanager.rmcontainer.ContainerAllocationExpirer; import org.apache.hadoop.yarn.server.resourcemanager.rmnode.RMNode; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.ResourceScheduler; - import org.apache.hadoop.yarn.server.resourcemanager.scheduler.distributed.QueueLimitCalculator; import org.apache.hadoop.yarn.server.resourcemanager.security.AMRMTokenSecretManager; import org.apache.hadoop.yarn.server.resourcemanager.security.ClientToAMTokenSecretManagerInRM; @@ -80,6 +80,8 @@ private QueueLimitCalculator queueLimitCalculator; + private MutableConfigurationManager mutableConfMgr; + /** * Default constructor. To be used in conjunction with setter methods for * individual fields. @@ -511,4 +513,14 @@ public void setRMAppLifetimeMonitor( public RMAppLifetimeMonitor getRMAppLifetimeMonitor() { return this.activeServiceContext.getRMAppLifetimeMonitor(); } + + @Override + public MutableConfigurationManager getMutableConfigurationManager() { + return this.mutableConfMgr; + } + + @Override + public void setMutableConfigurationManager(MutableConfigurationManager mutableConfMgr) { + this.mutableConfMgr = mutableConfMgr; + } } diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/conf/ConfigurationMutationPolicy.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/conf/ConfigurationMutationPolicy.java new file mode 100644 index 0000000..4e9e53a --- /dev/null +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/conf/ConfigurationMutationPolicy.java @@ -0,0 +1,11 @@ +package org.apache.hadoop.yarn.server.resourcemanager.conf; + +import java.util.Map; + +import org.apache.hadoop.security.UserGroupInformation; + +public interface ConfigurationMutationPolicy { + + public boolean applyPolicy(UserGroupInformation ugi, Map confUpdate); + +} diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/conf/MutableConfigurationManager.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/conf/MutableConfigurationManager.java new file mode 100644 index 0000000..9e874c4 --- /dev/null +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/conf/MutableConfigurationManager.java @@ -0,0 +1,53 @@ +package org.apache.hadoop.yarn.server.resourcemanager.conf; + +import java.util.HashMap; +import java.util.Map; + +import com.google.common.base.Joiner; +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.apache.hadoop.conf.Configurable; +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.security.UserGroupInformation; +import org.apache.hadoop.service.AbstractService; +import org.apache.hadoop.util.ReflectionUtils; +import org.apache.hadoop.yarn.conf.YarnConfiguration; +import org.apache.hadoop.yarn.exceptions.YarnRuntimeException; +import org.apache.hadoop.yarn.server.resourcemanager.conf.store.ConfigurationType; +import org.apache.hadoop.yarn.server.resourcemanager.conf.store.YarnConfigurationStore; +import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.ConfigurationMutationInfo; + +public class MutableConfigurationManager { + + private YarnConfigurationStore confStore; + private ConfigurationMutationPolicy mutationPolicy; + + private static final Log LOG = LogFactory.getLog(MutableConfigurationManager.class); + + public void init(Configuration conf) throws Exception { + // For now, only support mutations for capacity scheduler configuration. + Class policyClass = + conf.getClass(YarnConfiguration.RM_CONF_MUTATION_POLICY, CapacitySchedulerMutationPolicy.class) + .asSubclass(ConfigurationMutationPolicy.class); + this.mutationPolicy = ReflectionUtils.newInstance(policyClass, conf); + LOG.info("Initialized configuration mutation policy " + policyClass); + } + + public void setConfigurationStore(YarnConfigurationStore confStore) { + this.confStore = confStore; + } + + public void updateConfiguration(UserGroupInformation ugi, ConfigurationMutationInfo mutationInfo, + ConfigurationType mutationType) { + LOG.info("Add:" + Joiner.on('\n').join(mutationInfo.getAddQueueInfo())); + LOG.info("Remove:" + Joiner.on('\n').join(mutationInfo.getRemoveQueueInfo())); + LOG.info("Update:" + Joiner.on('\n').join(mutationInfo.getUpdateQueueInfo())); + //TODO flatten mutationInfo object to confUpdate + Map confUpdate = new HashMap<>(); + if (mutationPolicy.applyPolicy(ugi, confUpdate)) { + confStore.store(mutationType, confUpdate); + } else { + throw new YarnRuntimeException("Configuration update is invalid."); + } + } +} \ No newline at end of file