diff --git a/hadoop-tools/hadoop-archives/src/test/java/org/apache/hadoop/tools/TestHadoopArchives.java b/hadoop-tools/hadoop-archives/src/test/java/org/apache/hadoop/tools/TestHadoopArchives.java index 3fa5919..d609ed4 100644 --- a/hadoop-tools/hadoop-archives/src/test/java/org/apache/hadoop/tools/TestHadoopArchives.java +++ b/hadoop-tools/hadoop-archives/src/test/java/org/apache/hadoop/tools/TestHadoopArchives.java @@ -18,6 +18,10 @@ package org.apache.hadoop.tools; +import static org.junit.Assert.assertArrayEquals; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertTrue; + import java.io.ByteArrayOutputStream; import java.io.FilterInputStream; import java.io.IOException; @@ -43,11 +47,11 @@ import org.apache.hadoop.io.IOUtils; import org.apache.hadoop.util.JarFinder; import org.apache.hadoop.util.ToolRunner; +import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CSPlainConfiguration; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacitySchedulerConfiguration; import org.apache.log4j.Level; import org.junit.After; import org.junit.Assert; -import static org.junit.Assert.*; import org.junit.Before; import org.junit.Test; @@ -103,10 +107,10 @@ private static String createFile(Path root, FileSystem fs, byte[] fileContent, S @Before public void setUp() throws Exception { conf = new Configuration(); - conf.set(CapacitySchedulerConfiguration.PREFIX + conf.set(CSPlainConfiguration.PREFIX + CapacitySchedulerConfiguration.ROOT + "." - + CapacitySchedulerConfiguration.QUEUES, "default"); - conf.set(CapacitySchedulerConfiguration.PREFIX + + CSPlainConfiguration.QUEUES, "default"); + conf.set(CSPlainConfiguration.PREFIX + CapacitySchedulerConfiguration.ROOT + ".default." + CapacitySchedulerConfiguration.CAPACITY, "100"); dfscluster = new MiniDFSCluster diff --git a/hadoop-tools/hadoop-extras/src/test/java/org/apache/hadoop/tools/TestDistCh.java b/hadoop-tools/hadoop-extras/src/test/java/org/apache/hadoop/tools/TestDistCh.java index 79265e6..1512258 100644 --- a/hadoop-tools/hadoop-extras/src/test/java/org/apache/hadoop/tools/TestDistCh.java +++ b/hadoop-tools/hadoop-extras/src/test/java/org/apache/hadoop/tools/TestDistCh.java @@ -38,6 +38,7 @@ import org.apache.hadoop.hdfs.server.namenode.FSNamesystem; import org.apache.hadoop.io.IOUtils; import org.apache.hadoop.mapred.MiniMRClientClusterFactory; +import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CSPlainConfiguration; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacitySchedulerConfiguration; import org.apache.log4j.Level; @@ -130,9 +131,11 @@ Path mkdir(Path dir) throws IOException { public void testDistCh() throws Exception { final Configuration conf = new Configuration(); - conf.set(CapacitySchedulerConfiguration.PREFIX+CapacitySchedulerConfiguration.ROOT+"."+CapacitySchedulerConfiguration.QUEUES, "default"); - conf.set(CapacitySchedulerConfiguration.PREFIX+CapacitySchedulerConfiguration.ROOT+".default."+CapacitySchedulerConfiguration.CAPACITY, "100"); - final MiniDFSCluster cluster= new MiniDFSCluster.Builder(conf).numDataNodes(2).format(true).build(); + conf.set(CSPlainConfiguration.PREFIX + CSPlainConfiguration.ROOT + "." + + CSPlainConfiguration.QUEUES, "default"); + conf.set(CSPlainConfiguration.PREFIX + CSPlainConfiguration.ROOT + + ".default." + CSPlainConfiguration.CAPACITY, "100"); + final MiniDFSCluster cluster= new MiniDFSCluster.Builder(conf).numDataNodes(2).format(true).build(); final FileSystem fs = cluster.getFileSystem(); final FsShell shell = new FsShell(conf); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/api/impl/TestYarnClient.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/api/impl/TestYarnClient.java index 7e97134..6fefea4 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/api/impl/TestYarnClient.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/api/impl/TestYarnClient.java @@ -112,12 +112,11 @@ import org.apache.hadoop.yarn.server.resourcemanager.reservation.ReservationSystemTestUtil; import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMApp; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.ResourceScheduler; +import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CSPlainConfiguration; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacityScheduler; -import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacitySchedulerConfiguration; import org.apache.hadoop.yarn.util.Clock; import org.apache.hadoop.yarn.util.Records; import org.apache.hadoop.yarn.util.UTCClock; -import org.apache.hadoop.yarn.util.timeline.TimelineUtils; import org.apache.log4j.Level; import org.apache.log4j.LogManager; import org.apache.log4j.Logger; @@ -1072,7 +1071,7 @@ public void testParseTimelineDelegationTokenRenewer() throws Exception { @Test public void testReservationAPIs() { // initialize - CapacitySchedulerConfiguration conf = new CapacitySchedulerConfiguration(); + CSPlainConfiguration conf = new CSPlainConfiguration(); ReservationSystemTestUtil.setupQueueConfiguration(conf); conf.setClass(YarnConfiguration.RM_SCHEDULER, CapacityScheduler.class, ResourceScheduler.class); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/reservation/ReservationSchedulerConfiguration.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/reservation/ReservationSchedulerConfiguration.java index 2af1ffd..76aee95 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/reservation/ReservationSchedulerConfiguration.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/reservation/ReservationSchedulerConfiguration.java @@ -19,10 +19,9 @@ package org.apache.hadoop.yarn.server.resourcemanager.reservation; import org.apache.hadoop.classification.InterfaceAudience; -import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.yarn.api.records.ReservationDefinition; -public abstract class ReservationSchedulerConfiguration extends Configuration { +public interface ReservationSchedulerConfiguration { @InterfaceAudience.Private public static final long DEFAULT_RESERVATION_WINDOW = 24*60*60*1000; // 1 day in msec @@ -53,13 +52,6 @@ @InterfaceAudience.Private public static final float DEFAULT_CAPACITY_OVER_TIME_MULTIPLIER = 1; - public ReservationSchedulerConfiguration() { super(); } - - public ReservationSchedulerConfiguration( - Configuration configuration) { - super(configuration); - } - /** * Checks if the queue participates in reservation based scheduling * @param queue @@ -74,9 +66,7 @@ public ReservationSchedulerConfiguration( * @return length in time in milliseconds for which to check the * {@link SharingPolicy} */ - public long getReservationWindow(String queue) { - return DEFAULT_RESERVATION_WINDOW; - } + public long getReservationWindow(String queue); /** * Gets the average allowed capacity which will aggregated over the @@ -85,27 +75,21 @@ public long getReservationWindow(String queue) { * @param queue name of the queue * @return average capacity allowed by the {@link SharingPolicy} */ - public float getAverageCapacity(String queue) { - return DEFAULT_CAPACITY_OVER_TIME_MULTIPLIER; - } + public float getAverageCapacity(String queue); /** * Gets the maximum capacity at any time that the {@link SharingPolicy} allows * @param queue name of the queue * @return maximum allowed capacity at any time */ - public float getInstantaneousMaxCapacity(String queue) { - return DEFAULT_CAPACITY_OVER_TIME_MULTIPLIER; - } + public float getInstantaneousMaxCapacity(String queue); /** * Gets the name of the {@link SharingPolicy} class associated with the queue * @param queue name of the queue * @return the class name of the {@link SharingPolicy} */ - public String getReservationAdmissionPolicy(String queue) { - return DEFAULT_RESERVATION_ADMISSION_POLICY; - } + public String getReservationAdmissionPolicy(String queue); /** * Gets the name of the {@link ReservationAgent} class associated with the @@ -113,18 +97,14 @@ public String getReservationAdmissionPolicy(String queue) { * @param queue name of the queue * @return the class name of the {@link ReservationAgent} */ - public String getReservationAgent(String queue) { - return DEFAULT_RESERVATION_AGENT_NAME; - } + public String getReservationAgent(String queue); /** * Checks whether the reservation queues be hidden or visible * @param queuePath name of the queue * @return true if reservation queues should be visible */ - public boolean getShowReservationAsQueues(String queuePath) { - return DEFAULT_SHOW_RESERVATIONS_AS_QUEUES; - } + public boolean getShowReservationAsQueues(String queuePath); /** * Gets the name of the {@link Planner} class associated with the @@ -132,9 +112,7 @@ public boolean getShowReservationAsQueues(String queuePath) { * @param queue name of the queue * @return the class name of the {@link Planner} */ - public String getReplanner(String queue) { - return DEFAULT_RESERVATION_PLANNER_NAME; - } + public String getReplanner(String queue); /** * Gets whether the applications should be killed or moved to the parent queue @@ -143,9 +121,7 @@ public String getReplanner(String queue) { * @return true if application should be moved, false if they need to be * killed */ - public boolean getMoveOnExpiry(String queue) { - return DEFAULT_RESERVATION_MOVE_ON_EXPIRY; - } + public boolean getMoveOnExpiry(String queue); /** * Gets the time in milliseconds for which the {@link Planner} will verify @@ -153,7 +129,5 @@ public boolean getMoveOnExpiry(String queue) { * @param queue name of the queue * @return the time in milliseconds for which to check constraints */ - public long getEnforcementWindow(String queue) { - return DEFAULT_RESERVATION_ENFORCEMENT_WINDOW; - } + public long getEnforcementWindow(String queue); } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/AbstractCSQueue.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/AbstractCSQueue.java index eb7218b..b7e0683 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/AbstractCSQueue.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/AbstractCSQueue.java @@ -25,6 +25,7 @@ import org.apache.commons.lang.StringUtils; import org.apache.hadoop.classification.InterfaceAudience.Private; +import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.security.UserGroupInformation; import org.apache.hadoop.security.authorize.AccessControlList; import org.apache.hadoop.yarn.api.records.QueueACL; @@ -380,8 +381,9 @@ public ResourceUsage getQueueResourceUsage() { */ private boolean isQueueHierarchyPreemptionDisabled(CSQueue q) { CapacitySchedulerConfiguration csConf = csContext.getConfiguration(); + Configuration conf = csContext.getConf(); boolean systemWidePreemption = - csConf.getBoolean(YarnConfiguration.RM_SCHEDULER_ENABLE_MONITORS, + conf.getBoolean(YarnConfiguration.RM_SCHEDULER_ENABLE_MONITORS, YarnConfiguration.DEFAULT_RM_SCHEDULER_ENABLE_MONITORS); CSQueue parentQ = q.getParent(); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CSConfUtils.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CSConfUtils.java new file mode 100644 index 0000000..58573a0 --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CSConfUtils.java @@ -0,0 +1,204 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + + +package org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity; + +import java.util.ArrayList; +import java.util.Collection; +import java.util.Collections; +import java.util.HashSet; +import java.util.List; +import java.util.Locale; +import java.util.Set; +import java.util.StringTokenizer; + +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.apache.hadoop.yarn.api.records.QueueACL; +import org.apache.hadoop.yarn.api.records.Resource; +import org.apache.hadoop.yarn.server.resourcemanager.nodelabels.RMNodeLabelsManager; +import org.apache.hadoop.yarn.util.resource.Resources; + +import com.google.common.collect.ImmutableSet; + +public class CSConfUtils { + private static final Log LOG = + LogFactory.getLog(CSConfUtils.class); + + public static Set getAccessibleNodeLabels(String queue, + String accessibleLabelStr) { + // When accessible-label is null, + if (accessibleLabelStr == null) { + // Only return null when queue is not ROOT + if (!queue.equals(CapacitySchedulerConfiguration.ROOT)) { + return null; + } + } else { + // print a warning when accessibleNodeLabel specified in config and queue + // is ROOT + if (queue.equals(CapacitySchedulerConfiguration.ROOT)) { + LOG.warn("Accessible node labels for root queue will be ignored," + + " it will be automatically set to \"*\"."); + } + } + + // always return ANY for queue root + if (queue.equals(CapacitySchedulerConfiguration.ROOT)) { + return ImmutableSet.of(RMNodeLabelsManager.ANY); + } + + // In other cases, split the accessibleLabelStr by "," + Set set = new HashSet(); + for (String str : accessibleLabelStr.split(",")) { + if (!str.trim().isEmpty()) { + set.add(str.trim()); + } + } + + // if labels contains "*", only keep ANY behind + if (set.contains(RMNodeLabelsManager.ANY)) { + set.clear(); + set.add(RMNodeLabelsManager.ANY); + } + return Collections.unmodifiableSet(set); + } + + public static float getAndCheckLabeledQueueCapacity(String queue, + String label, float capacity, float defaultValue) { + if (capacity < CapacitySchedulerConfiguration.MINIMUM_CAPACITY_VALUE + || capacity > CapacitySchedulerConfiguration.MAXIMUM_CAPACITY_VALUE) { + throw new IllegalArgumentException("Illegal capacity of " + capacity + + " for node-label=" + label + " in queue=" + queue + + ", valid capacity should in range of [0, 100]."); + } + if (LOG.isDebugEnabled()) { + LOG.debug("CSConf - getCapacityOfLabel: prefix=" + queue + ", label=" + + label + ", capacity=" + capacity); + } + return capacity; + } + + public static String getAclKey(QueueACL acl) { + return "acl_" + acl.toString().toLowerCase(Locale.ENGLISH); + } + + public static Resource getMaximumAllocationPerQueue(String queue, + int maxAllocationMbPerQueue, int maxAllocationVcoresPerQueue, + Resource clusterMaxAllocation) { + if (LOG.isDebugEnabled()) { + LOG.debug("max alloc mb per queue for " + queue + " is " + + maxAllocationMbPerQueue); + LOG.debug("max alloc vcores per queue for " + queue + " is " + + maxAllocationVcoresPerQueue); + } + if (maxAllocationMbPerQueue == (int) CapacitySchedulerConfiguration.UNDEFINED) { + LOG.info("max alloc mb per queue for " + queue + " is undefined"); + maxAllocationMbPerQueue = clusterMaxAllocation.getMemory(); + } + if (maxAllocationVcoresPerQueue == (int) CapacitySchedulerConfiguration.UNDEFINED) { + LOG.info("max alloc vcore per queue for " + queue + " is undefined"); + maxAllocationVcoresPerQueue = clusterMaxAllocation.getVirtualCores(); + } + Resource result = + Resources.createResource(maxAllocationMbPerQueue, + maxAllocationVcoresPerQueue); + if (maxAllocationMbPerQueue > clusterMaxAllocation.getMemory() + || maxAllocationVcoresPerQueue > clusterMaxAllocation.getVirtualCores()) { + throw new IllegalArgumentException( + "Queue maximum allocation cannot be larger than the cluster setting" + + " for queue " + queue + " max allocation per queue: " + result + + " cluster setting: " + clusterMaxAllocation); + } + return result; + } + + /** + * Returns a collection of strings, trimming leading and trailing whitespeace + * on each value + * + * @param str + * String to parse + * @param delim + * delimiter to separate the values + * @return Collection of parsed elements. + */ + private static Collection getTrimmedStringCollection(String str, + String delim) { + List values = new ArrayList(); + if (str == null) + return values; + StringTokenizer tokenizer = new StringTokenizer(str, delim); + while (tokenizer.hasMoreTokens()) { + String next = tokenizer.nextToken(); + if (next == null || next.trim().isEmpty()) { + continue; + } + values.add(next.trim()); + } + return values; + } + + /** + * Get user/group mappings to queues. + * + * @return user/groups mappings or null on illegal configs + */ + public static List getQueueMappings(String mappingFromConf) { + List mappings = + new ArrayList(); + Collection mappingsString = + getTrimmedStringCollection(mappingFromConf, ","); + for (String mappingValue : mappingsString) { + String[] mapping = + getTrimmedStringCollection(mappingValue, ":") + .toArray(new String[] {}); + if (mapping.length != 3 || mapping[1].length() == 0 + || mapping[2].length() == 0) { + throw new IllegalArgumentException( + "Illegal queue mapping " + mappingValue); + } + + QueueMapping m; + try { + QueueMapping.MappingType mappingType; + if (mapping[0].equals("u")) { + mappingType = QueueMapping.MappingType.USER; + } else if (mapping[0].equals("g")) { + mappingType = QueueMapping.MappingType.GROUP; + } else { + throw new IllegalArgumentException( + "unknown mapping prefix " + mapping[0]); + } + m = new QueueMapping( + mappingType, + mapping[1], + mapping[2]); + } catch (Throwable t) { + throw new IllegalArgumentException( + "Illegal queue mapping " + mappingValue); + } + + if (m != null) { + mappings.add(m); + } + } + + return mappings; + } +} diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CSHierarchyConfiguration.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CSHierarchyConfiguration.java new file mode 100644 index 0000000..e221901 --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CSHierarchyConfiguration.java @@ -0,0 +1,514 @@ +/** +* Licensed to the Apache Software Foundation (ASF) under one +* or more contributor license agreements. See the NOTICE file +* distributed with this work for additional information +* regarding copyright ownership. The ASF licenses this file +* to you under the Apache License, Version 2.0 (the +* "License"); you may not use this file except in compliance +* with the License. You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, software +* distributed under the License is distributed on an "AS IS" BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions and +* limitations under the License. +*/ + +package org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity; + +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.Set; + +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.security.authorize.AccessControlList; +import org.apache.hadoop.util.ReflectionUtils; +import org.apache.hadoop.util.StringUtils; +import org.apache.hadoop.yarn.api.records.QueueACL; +import org.apache.hadoop.yarn.api.records.QueueState; +import org.apache.hadoop.yarn.api.records.Resource; +import org.apache.hadoop.yarn.conf.YarnConfiguration; +import org.apache.hadoop.yarn.security.AccessType; +import org.apache.hadoop.yarn.server.resourcemanager.reservation.ReservationSchedulerConfiguration; +import org.apache.hadoop.yarn.server.resourcemanager.scheduler.SchedulerUtils; +import org.apache.hadoop.yarn.server.resourcemanager.scheduler.config.BaseSchedulerHierarchyConfiguration; +import org.apache.hadoop.yarn.server.resourcemanager.scheduler.config.SchedulerConfNode; +import org.apache.hadoop.yarn.util.resource.ResourceCalculator; +import org.apache.hadoop.yarn.util.resource.Resources; + +public class CSHierarchyConfiguration extends + BaseSchedulerHierarchyConfiguration implements + CapacitySchedulerConfiguration { + private static final String LABELS_TAG = "LABELS"; + private static final String LABEL_TAG = "LABELS"; + + private static final Log LOG = + LogFactory.getLog(CSHierarchyConfiguration.class); + + public CSHierarchyConfiguration(SchedulerConfNode root) { + super(root); + } + + @Override + public int getMaximumSystemApplications() { + return getGlobalConfig(MAXIMUM_APPLICATIONS, + DEFAULT_MAXIMUM_SYSTEM_APPLICATIIONS); + } + + @Override + public float getMaximumApplicationMasterResourcePercent() { + return getGlobalConfig(MAXIMUM_AM_RESOURCE, + DEFAULT_MAXIMUM_APPLICATIONMASTERS_RESOURCE_PERCENT); + } + + @Override + public int getMaximumApplicationsPerQueue(String queue) { + return getQueueConfig(queue, MAXIMUM_APPLICATIONS, (int)UNDEFINED); + } + + @Override + public float getMaximumApplicationMasterResourcePerQueuePercent(String queue) { + return getQueueConfig(queue, MAXIMUM_AM_RESOURCE, + getMaximumApplicationMasterResourcePercent()); + } + + @Override + public float getNonLabeledQueueCapacity(String queue) { + float capacity = queue.equals("root") ? 100.0f : + getQueueConfig(queue, CAPACITY, UNDEFINED); + if (capacity < MINIMUM_CAPACITY_VALUE || capacity > MAXIMUM_CAPACITY_VALUE) { + throw new IllegalArgumentException("Illegal " + + "capacity of " + capacity + " for queue " + queue); + } + LOG.debug("CSConf - getCapacity: queuePrefix=" + queue + + ", capacity=" + capacity); + return capacity; + } + + @Override + public void setCapacity(String queue, float capacity) { + if (queue.equals("root")) { + throw new IllegalArgumentException( + "Cannot set capacity, root queue has a fixed capacity of 100.0f"); + } + SchedulerConfNode node = getQueueNode(queue); + node.setValue(CAPACITY, String.valueOf(capacity)); + LOG.debug("CSConf - setCapacity: queuePrefix=" + queue + + ", capacity=" + capacity); + } + + @Override + public float getNonLabeledQueueMaximumCapacity(String queue) { + float maxCapacity = + getQueueConfig(queue, MAXIMUM_CAPACITY, MAXIMUM_CAPACITY_VALUE); + maxCapacity = (maxCapacity == DEFAULT_MAXIMUM_CAPACITY_VALUE) ? + MAXIMUM_CAPACITY_VALUE : maxCapacity; + return maxCapacity; + } + + @Override + public void setMaximumCapacity(String queue, float maxCapacity) { + if (maxCapacity > MAXIMUM_CAPACITY_VALUE) { + throw new IllegalArgumentException("Illegal " + "maximum-capacity of " + + maxCapacity + " for queue " + queue); + } + SchedulerConfNode node = getQueueNode(queue); + node.setValue(MAXIMUM_CAPACITY, String.valueOf(maxCapacity)); + LOG.debug("CSConf - setMaxCapacity: queuePrefix=" + queue + + ", maxCapacity=" + maxCapacity); + } + + /** + * Get queue label node, will throw exception when queue node doesn't exist, + * but will return null when label node doesn't exist + */ + private SchedulerConfNode getQueueLabelNode(String queue, String label) { + SchedulerConfNode queueNode = getQueueNode(queue); + SchedulerConfNode labelsNode = queueNode.getChild(LABELS_TAG); + if (null == labelsNode) { + return null; + } + SchedulerConfNode labelNode = labelsNode.getChild(LABEL_TAG, label); + return labelNode; + } + + private SchedulerConfNode getAndCreateQueueLabelNodeIfNotExisted( + String queue, String label) { + SchedulerConfNode queueNode = getQueueNode(queue); + + // Add labels node + SchedulerConfNode labelsNode = queueNode.getChild(LABELS_TAG); + if (null == labelsNode) { + labelsNode = queueNode.addChild(LABELS_TAG, "", ""); + } + + // Add label node + SchedulerConfNode labelNode = labelsNode.getChild(LABEL_TAG, label); + if (null == labelNode) { + labelNode = labelsNode.addChild(LABEL_TAG, label, ""); + } + + return labelNode; + } + + @Override + public void setCapacityByLabel(String queue, String label, float capacity) { + SchedulerConfNode labelNode = getAndCreateQueueLabelNodeIfNotExisted(queue, label); + labelNode.setValue(CAPACITY, String.valueOf(capacity)); + } + + @Override + public void setMaximumCapacityByLabel(String queue, String label, + float capacity) { + SchedulerConfNode labelNode = getAndCreateQueueLabelNodeIfNotExisted(queue, label); + labelNode.setValue(MAXIMUM_CAPACITY, String.valueOf(capacity)); + } + + @Override + public int getUserLimit(String queue) { + return getQueueConfig(queue, USER_LIMIT, DEFAULT_USER_LIMIT); + } + + @Override + public void setUserLimit(String queue, int userLimit) { + SchedulerConfNode queueNode = getQueueNode(queue); + queueNode.setValue(USER_LIMIT, String.valueOf(userLimit)); + } + + @Override + public float getUserLimitFactor(String queue) { + return getQueueConfig(queue, USER_LIMIT_FACTOR, DEFAULT_USER_LIMIT_FACTOR); + } + + @Override + public void setUserLimitFactor(String queue, float userLimitFactor) { + getQueueNode(queue).setValue(USER_LIMIT_FACTOR, + String.valueOf(userLimitFactor)); + } + + @Override + public QueueState getState(String queue) { + String state = getQueueConfig(queue, STATE, QueueState.RUNNING.name()); + return QueueState.valueOf(state); + } + + @Override + public void setAccessibleNodeLabels(String queue, Set labels) { + if (labels == null) { + return; + } + String str = StringUtils.join(",", labels); + getQueueNode(queue).setValue(ACCESSIBLE_NODE_LABELS, str); + } + + @Override + public Set getAccessibleNodeLabels(String queue) { + String accessibleLabelStr = getQueueConfig(queue, ACCESSIBLE_NODE_LABELS, null); + return CSConfUtils.getAccessibleNodeLabels(queue, accessibleLabelStr); + } + + private float internalGetLabeledQueueCapacity(String queue, String label, + String suffix, float defaultValue) { + SchedulerConfNode labelNode = getQueueLabelNode(queue, label); + float capacity = + Float.parseFloat(labelNode.getValue(suffix, String.valueOf(0))); + return CSConfUtils.getAndCheckLabeledQueueCapacity(queue, label, capacity, + defaultValue); + } + + @Override + public float getLabeledQueueCapacity(String queue, String label) { + return internalGetLabeledQueueCapacity(queue, label, CAPACITY, 0f); + } + + @Override + public float getLabeledQueueMaximumCapacity(String queue, String label) { + return internalGetLabeledQueueCapacity(queue, label, MAXIMUM_CAPACITY, 100f); + } + + @Override + public String getDefaultNodeLabelExpression(String queue) { + return getQueueConfig(queue, DEFAULT_NODE_LABEL_EXPRESSION, null); + } + + @Override + public void setDefaultNodeLabelExpression(String queue, String exp) { + getQueueNode(queue).setValue(DEFAULT_NODE_LABEL_EXPRESSION, exp); + } + + @Override + public boolean getReservationContinueLook() { + return getGlobalConfig(RESERVE_CONT_LOOK_ALL_NODES_SUFFIX, + DEFAULT_RESERVE_CONT_LOOK_ALL_NODES); + } + + @Override + public AccessControlList getAcl(String queue, QueueACL acl) { + // The root queue defaults to all access if not defined + // Sub queues inherit access if not defined + String defaultAcl = queue.equals(ROOT) ? ALL_ACL : NONE_ACL; + String aclString = + getQueueConfig(queue, CSConfUtils.getAclKey(acl), defaultAcl); + return new AccessControlList(aclString); + } + + @Override + public void setAcl(String queue, QueueACL acl, String aclString) { + getQueueNode(queue).setValue(CSConfUtils.getAclKey(acl), aclString); + } + + @Override + public Map getAcls(String queue) { + Map acls = + new HashMap(); + for (QueueACL acl : QueueACL.values()) { + acls.put(SchedulerUtils.toAccessType(acl), getAcl(queue, acl)); + } + return acls; + } + + @Override + public void setAcls(String queue, Map acls) { + for (Map.Entry e : acls.entrySet()) { + setAcl(queue, e.getKey(), e.getValue().getAclString()); + } + } + + @Override + public String[] getQueues(String queue) { + return getSubQueueNames(queue).toArray(new String[0]); + } + + @Override + public void setQueues(String queue, String[] subQueues) { + addQueues(queue, subQueues); + } + + @Override + public Resource getMinimumAllocation() { + int minimumMemory = getGlobalConfig( + YarnConfiguration.RM_SCHEDULER_MINIMUM_ALLOCATION_MB, + YarnConfiguration.DEFAULT_RM_SCHEDULER_MINIMUM_ALLOCATION_MB); + int minimumCores = getGlobalConfig( + YarnConfiguration.RM_SCHEDULER_MINIMUM_ALLOCATION_VCORES, + YarnConfiguration.DEFAULT_RM_SCHEDULER_MINIMUM_ALLOCATION_VCORES); + return Resources.createResource(minimumMemory, minimumCores); + } + + @Override + public Resource getMaximumAllocation() { + int maximumMemory = getGlobalConfig( + YarnConfiguration.RM_SCHEDULER_MAXIMUM_ALLOCATION_MB, + YarnConfiguration.DEFAULT_RM_SCHEDULER_MAXIMUM_ALLOCATION_MB); + int maximumCores = getGlobalConfig( + YarnConfiguration.RM_SCHEDULER_MAXIMUM_ALLOCATION_VCORES, + YarnConfiguration.DEFAULT_RM_SCHEDULER_MAXIMUM_ALLOCATION_VCORES); + return Resources.createResource(maximumMemory, maximumCores); + } + + @Override + public Resource getMaximumAllocationPerQueue(String queue) { + int maxAllocationMbPerQueue = + getQueueConfig(queue, MAXIMUM_ALLOCATION_MB, (int) UNDEFINED); + int maxAllocationVcoresPerQueue = + getQueueConfig(queue, MAXIMUM_ALLOCATION_VCORES, (int) UNDEFINED); + Resource clusterMax = getMaximumAllocation(); + return CSConfUtils.getMaximumAllocationPerQueue(queue, + maxAllocationMbPerQueue, maxAllocationVcoresPerQueue, clusterMax); + } + + @Override + public boolean getEnableUserMetrics() { + return getGlobalConfig(ENABLE_USER_METRICS, DEFAULT_ENABLE_USER_METRICS); + } + + @Override + public int getNodeLocalityDelay() { + int delay = getGlobalConfig(NODE_LOCALITY_DELAY, DEFAULT_NODE_LOCALITY_DELAY); + return (delay == DEFAULT_NODE_LOCALITY_DELAY) ? 0 : delay; + } + + @Override + public ResourceCalculator getResourceCalculator() { + // Since there's bunch of class-related utilities in Configuration, here + // will reuse Hadoop Configuration + String className = + getGlobalConfig(RESOURCE_CALCULATOR_CLASS, + DEFAULT_RESOURCE_CALCULATOR_CLASS.getCanonicalName()); + + Configuration tmpConf = new Configuration(false); + tmpConf.set(RESOURCE_CALCULATOR_CLASS, className); + return ReflectionUtils.newInstance( + tmpConf.getClass( + RESOURCE_CALCULATOR_CLASS, + DEFAULT_RESOURCE_CALCULATOR_CLASS, + ResourceCalculator.class), + tmpConf); + } + + @Override + public void setResourceCalculator( + Class resourceCalculatorClass) { + // Since there's bunch of class-related utilities in Configuration, here + // will reuse Hadoop Configuration + Configuration tmpConf = new Configuration(false); + tmpConf.setClass(RESOURCE_CALCULATOR_CLASS, resourceCalculatorClass, + ResourceCalculator.class); + setGlobalConfig(RESOURCE_CALCULATOR_CLASS, + resourceCalculatorClass.getCanonicalName()); + } + + @Override + public boolean getScheduleAynschronously() { + return getGlobalConfig(SCHEDULE_ASYNCHRONOUSLY_ENABLE, + DEFAULT_SCHEDULE_ASYNCHRONOUSLY_ENABLE); + } + + @Override + public void setScheduleAynschronously(boolean async) { + setGlobalConfig(SCHEDULE_ASYNCHRONOUSLY_ENABLE, + String.valueOf(DEFAULT_SCHEDULE_ASYNCHRONOUSLY_ENABLE)); + } + + @Override + public boolean getOverrideWithQueueMappings() { + return getGlobalConfig(ENABLE_QUEUE_MAPPING_OVERRIDE, + DEFAULT_ENABLE_QUEUE_MAPPING_OVERRIDE); + } + + @Override + public List getQueueMappings() { + return CSConfUtils.getQueueMappings(getGlobalConfig(QUEUE_MAPPING, null)); + } + + @Override + public boolean isReservable(String queue) { + return getQueueConfig(queue, IS_RESERVABLE, false); + } + + @Override + public void setReservable(String queue, boolean isReservable) { + SchedulerConfNode queueNode = getQueueNode(queue); + queueNode.setValue(IS_RESERVABLE, String.valueOf(isReservable)); + } + + @Override + public long getReservationWindow(String queue) { + return getQueueConfig(queue, RESERVATION_WINDOW, + ReservationSchedulerConfiguration.DEFAULT_RESERVATION_WINDOW); + } + + @Override + public float getAverageCapacity(String queue) { + return getQueueConfig(queue, AVERAGE_CAPACITY, MAXIMUM_CAPACITY_VALUE); + } + + @Override + public float getInstantaneousMaxCapacity(String queue) { + return getQueueConfig(queue, INSTANTANEOUS_MAX_CAPACITY, + MAXIMUM_CAPACITY_VALUE); + } + + @Override + public void setInstantaneousMaxCapacity(String queue, float instMaxCapacity) { + SchedulerConfNode queueNode = getQueueNode(queue); + queueNode.setValue(INSTANTANEOUS_MAX_CAPACITY, String.valueOf(instMaxCapacity)); + } + + @Override + public void setReservationWindow(String queue, long reservationWindow) { + SchedulerConfNode queueNode = getQueueNode(queue); + queueNode.setValue(RESERVATION_WINDOW, String + .valueOf(ReservationSchedulerConfiguration.DEFAULT_RESERVATION_WINDOW)); + } + + @Override + public void setAverageCapacity(String queue, float avgCapacity) { + SchedulerConfNode queueNode = getQueueNode(queue); + queueNode.setValue(AVERAGE_CAPACITY, String.valueOf(MAXIMUM_CAPACITY_VALUE)); + } + + @Override + public String getReservationAdmissionPolicy(String queue) { + return getQueueConfig(queue, RESERVATION_ADMISSION_POLICY, + DEFAULT_RESERVATION_ADMISSION_POLICY); + } + + @Override + public void setReservationAdmissionPolicy(String queue, + String reservationPolicy) { + getQueueNode(queue).setValue(RESERVATION_ADMISSION_POLICY, + DEFAULT_RESERVATION_ADMISSION_POLICY); + } + + @Override + public String getReservationAgent(String queue) { + return getQueueConfig(queue, RESERVATION_ADMISSION_POLICY, + DEFAULT_RESERVATION_ADMISSION_POLICY); + } + + @Override + public void setReservationAgent(String queue, String reservationPolicy) { + getQueueNode(queue).setValue(RESERVATION_AGENT_NAME, + reservationPolicy); + } + + @Override + public boolean getShowReservationAsQueues(String queuePath) { + return getQueueConfig(queuePath, RESERVATION_SHOW_RESERVATION_AS_QUEUE, + ReservationSchedulerConfiguration.DEFAULT_SHOW_RESERVATIONS_AS_QUEUES); + } + + @Override + public String getReplanner(String queue) { + return getQueueConfig(queue, RESERVATION_PLANNER_NAME, + DEFAULT_RESERVATION_PLANNER_NAME); + } + + @Override + public boolean getMoveOnExpiry(String queue) { + return getQueueConfig(queue, RESERVATION_MOVE_ON_EXPIRY, + ReservationSchedulerConfiguration.DEFAULT_RESERVATION_MOVE_ON_EXPIRY); + } + + @Override + public long getEnforcementWindow(String queue) { + return getQueueConfig( + queue, + RESERVATION_ENFORCEMENT_WINDOW, + ReservationSchedulerConfiguration.DEFAULT_RESERVATION_ENFORCEMENT_WINDOW); + } + + @Override + public void setPreemptionDisabled(String queue, boolean preemptionDisabled) { + getQueueNode(queue).setValue(QUEUE_PREEMPTION_DISABLED, + String.valueOf(preemptionDisabled)); + } + + @Override + public boolean getPreemptionDisabled(String queue, boolean defaultVal) { + return getQueueConfig(queue, QUEUE_PREEMPTION_DISABLED, defaultVal); + } + + @Override + public long getAsyncSchedulerIntervalMs() { + return getGlobalConfig(ASYNC_SCHEDULER_INTERVAL, + DEFAULT_ASYNC_SCHEDULER_INTERVAL); + } + + @Override + public void setMaximumApplicationsPerQueue(String queue, int value) { + getQueueNode(queue).setValue(MAXIMUM_APPLICATIONS, String.valueOf(value)); + } + + @Override + public void setMaximumApplicationMasterResourcePerQueuePercent(String queue, + float value) { + getQueueNode(queue).setValue(MAXIMUM_AM_RESOURCE, String.valueOf(value)); + } +} diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CSPlainConfiguration.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CSPlainConfiguration.java new file mode 100644 index 0000000..9df720a --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CSPlainConfiguration.java @@ -0,0 +1,714 @@ +/** +* Licensed to the Apache Software Foundation (ASF) under one +* or more contributor license agreements. See the NOTICE file +* distributed with this work for additional information +* regarding copyright ownership. The ASF licenses this file +* to you under the Apache License, Version 2.0 (the +* "License"); you may not use this file except in compliance +* with the License. You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, software +* distributed under the License is distributed on an "AS IS" BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions and +* limitations under the License. +*/ + +package org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity; + +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Locale; +import java.util.Map; +import java.util.Set; + +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.apache.hadoop.classification.InterfaceAudience.Private; +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.security.authorize.AccessControlList; +import org.apache.hadoop.util.ReflectionUtils; +import org.apache.hadoop.util.StringUtils; +import org.apache.hadoop.yarn.api.records.QueueACL; +import org.apache.hadoop.yarn.api.records.QueueState; +import org.apache.hadoop.yarn.api.records.Resource; +import org.apache.hadoop.yarn.conf.YarnConfiguration; +import org.apache.hadoop.yarn.nodelabels.CommonNodeLabelsManager; +import org.apache.hadoop.yarn.security.AccessType; +import org.apache.hadoop.yarn.server.resourcemanager.reservation.ReservationSchedulerConfiguration; +import org.apache.hadoop.yarn.server.resourcemanager.scheduler.SchedulerUtils; +import org.apache.hadoop.yarn.util.resource.DefaultResourceCalculator; +import org.apache.hadoop.yarn.util.resource.ResourceCalculator; +import org.apache.hadoop.yarn.util.resource.Resources; + +public class CSPlainConfiguration extends Configuration + implements CapacitySchedulerConfiguration, ReservationSchedulerConfiguration { + + private static final Log LOG = + LogFactory.getLog(CSPlainConfiguration.class); + + private static final String CS_CONFIGURATION_FILE = "capacity-scheduler.xml"; + + @Private + public static final String PREFIX = "yarn.scheduler.capacity."; + + @Private + public static final String DOT = "."; + + @Private + public static final String MAXIMUM_APPLICATIONS_SUFFIX = + "maximum-applications"; + + @Private + public static final String MAXIMUM_SYSTEM_APPLICATIONS = + PREFIX + MAXIMUM_APPLICATIONS_SUFFIX; + + @Private + public static final String MAXIMUM_AM_RESOURCE_SUFFIX = + "maximum-am-resource-percent"; + + @Private + public static final String MAXIMUM_APPLICATION_MASTERS_RESOURCE_PERCENT = + PREFIX + MAXIMUM_AM_RESOURCE_SUFFIX; + + @Private + public static final String QUEUES = "queues"; + + @Private + public static final String CAPACITY = "capacity"; + + @Private + public static final String MAXIMUM_CAPACITY = "maximum-capacity"; + + @Private + public static final String USER_LIMIT = "minimum-user-limit-percent"; + + @Private + public static final String USER_LIMIT_FACTOR = "user-limit-factor"; + + @Private + public static final String STATE = "state"; + + @Private + public static final String ACCESSIBLE_NODE_LABELS = "accessible-node-labels"; + + @Private + public static final String DEFAULT_NODE_LABEL_EXPRESSION = + "default-node-label-expression"; + + public static final String RESERVE_CONT_LOOK_ALL_NODES = PREFIX + + "reservations-continue-look-all-nodes"; + + @Private + public static final boolean DEFAULT_RESERVE_CONT_LOOK_ALL_NODES = true; + + @Private + public static final String MAXIMUM_ALLOCATION_MB = "maximum-allocation-mb"; + + @Private + public static final String MAXIMUM_ALLOCATION_VCORES = + "maximum-allocation-vcores"; + + @Private + public static final int DEFAULT_MAXIMUM_SYSTEM_APPLICATIIONS = 10000; + + @Private + public static final float + DEFAULT_MAXIMUM_APPLICATIONMASTERS_RESOURCE_PERCENT = 0.1f; + + @Private + public static final float UNDEFINED = -1; + + @Private + public static final float MINIMUM_CAPACITY_VALUE = 0; + + @Private + public static final float MAXIMUM_CAPACITY_VALUE = 100; + + @Private + public static final float DEFAULT_MAXIMUM_CAPACITY_VALUE = -1.0f; + + @Private + public static final int DEFAULT_USER_LIMIT = 100; + + @Private + public static final float DEFAULT_USER_LIMIT_FACTOR = 1.0f; + + @Private + public static final String ALL_ACL = "*"; + + @Private + public static final String NONE_ACL = " "; + + @Private public static final String ENABLE_USER_METRICS = + PREFIX +"user-metrics.enable"; + @Private public static final boolean DEFAULT_ENABLE_USER_METRICS = false; + + /** ResourceComparator for scheduling. */ + @Private public static final String RESOURCE_CALCULATOR_CLASS = + PREFIX + "resource-calculator"; + + @Private public static final Class + DEFAULT_RESOURCE_CALCULATOR_CLASS = DefaultResourceCalculator.class; + + @Private + public static final String ROOT = "root"; + + @Private + public static final String NODE_LOCALITY_DELAY = + PREFIX + "node-locality-delay"; + + @Private + public static final int DEFAULT_NODE_LOCALITY_DELAY = -1; + + @Private + public static final String SCHEDULE_ASYNCHRONOUSLY_PREFIX = + PREFIX + "schedule-asynchronously"; + + @Private + public static final String SCHEDULE_ASYNCHRONOUSLY_ENABLE = + SCHEDULE_ASYNCHRONOUSLY_PREFIX + ".enable"; + + @Private + public static final boolean DEFAULT_SCHEDULE_ASYNCHRONOUSLY_ENABLE = false; + + @Private + public static final String QUEUE_MAPPING = PREFIX + "queue-mappings"; + + @Private + public static final String ENABLE_QUEUE_MAPPING_OVERRIDE = QUEUE_MAPPING + "-override.enable"; + + @Private + public static final boolean DEFAULT_ENABLE_QUEUE_MAPPING_OVERRIDE = false; + + @Private + public static final String QUEUE_PREEMPTION_DISABLED = "disable_preemption"; + + @Private + public static final String AVERAGE_CAPACITY = "average-capacity"; + + @Private + public static final String IS_RESERVABLE = "reservable"; + + @Private + public static final String RESERVATION_WINDOW = "reservation-window"; + + @Private + public static final String INSTANTANEOUS_MAX_CAPACITY = + "instantaneous-max-capacity"; + + @Private + public static final String RESERVATION_ADMISSION_POLICY = + "reservation-policy"; + + @Private + public static final String RESERVATION_AGENT_NAME = "reservation-agent"; + + @Private + public static final String RESERVATION_SHOW_RESERVATION_AS_QUEUE = + "show-reservations-as-queues"; + + @Private + public static final String RESERVATION_PLANNER_NAME = "reservation-planner"; + + @Private + public static final String RESERVATION_MOVE_ON_EXPIRY = + "reservation-move-on-expiry"; + + @Private + public static final String RESERVATION_ENFORCEMENT_WINDOW = + "reservation-enforcement-window"; + + @Private + public static final String ASYNC_SCHEDULER_INTERVAL = + CSPlainConfiguration.SCHEDULE_ASYNCHRONOUSLY_PREFIX + + ".scheduling-interval-ms"; + + public CSPlainConfiguration() { + this(new Configuration()); + } + + public CSPlainConfiguration(Configuration configuration) { + this(configuration, true); + } + + public CSPlainConfiguration(Configuration configuration, + boolean useLocalConfigurationProvider) { + super(configuration); + if (useLocalConfigurationProvider) { + addResource(CS_CONFIGURATION_FILE); + } + } + + static String getQueuePrefix(String queue) { + String queueName = PREFIX + queue + DOT; + return queueName; + } + + private String getNodeLabelPrefix(String queue, String label) { + if (label.equals(CommonNodeLabelsManager.NO_LABEL)) { + return getQueuePrefix(queue); + } + return getQueuePrefix(queue) + ACCESSIBLE_NODE_LABELS + DOT + label + DOT; + } + + public int getMaximumSystemApplications() { + int maxApplications = + getInt(MAXIMUM_SYSTEM_APPLICATIONS, DEFAULT_MAXIMUM_SYSTEM_APPLICATIIONS); + return maxApplications; + } + + public float getMaximumApplicationMasterResourcePercent() { + return getFloat(MAXIMUM_APPLICATION_MASTERS_RESOURCE_PERCENT, + DEFAULT_MAXIMUM_APPLICATIONMASTERS_RESOURCE_PERCENT); + } + + + /** + * Get the maximum applications per queue setting. + * @param queue name of the queue + * @return setting specified or -1 if not set + */ + public int getMaximumApplicationsPerQueue(String queue) { + int maxApplicationsPerQueue = + getInt(getQueuePrefix(queue) + MAXIMUM_APPLICATIONS_SUFFIX, + (int)UNDEFINED); + return maxApplicationsPerQueue; + } + + /** + * Get the maximum am resource percent per queue setting. + * @param queue name of the queue + * @return per queue setting or defaults to the global am-resource-percent + * setting if per queue setting not present + */ + public float getMaximumApplicationMasterResourcePerQueuePercent(String queue) { + return getFloat(getQueuePrefix(queue) + MAXIMUM_AM_RESOURCE_SUFFIX, + getMaximumApplicationMasterResourcePercent()); + } + + public float getNonLabeledQueueCapacity(String queue) { + float capacity = queue.equals("root") ? 100.0f : getFloat( + getQueuePrefix(queue) + CAPACITY, UNDEFINED); + if (capacity < MINIMUM_CAPACITY_VALUE || capacity > MAXIMUM_CAPACITY_VALUE) { + throw new IllegalArgumentException("Illegal " + + "capacity of " + capacity + " for queue " + queue); + } + LOG.debug("CSConf - getCapacity: queuePrefix=" + getQueuePrefix(queue) + + ", capacity=" + capacity); + return capacity; + } + + public void setCapacity(String queue, float capacity) { + if (queue.equals("root")) { + throw new IllegalArgumentException( + "Cannot set capacity, root queue has a fixed capacity of 100.0f"); + } + setFloat(getQueuePrefix(queue) + CAPACITY, capacity); + LOG.debug("CSConf - setCapacity: queuePrefix=" + getQueuePrefix(queue) + + ", capacity=" + capacity); + } + + public float getNonLabeledQueueMaximumCapacity(String queue) { + float maxCapacity = getFloat(getQueuePrefix(queue) + MAXIMUM_CAPACITY, + MAXIMUM_CAPACITY_VALUE); + maxCapacity = (maxCapacity == DEFAULT_MAXIMUM_CAPACITY_VALUE) ? + MAXIMUM_CAPACITY_VALUE : maxCapacity; + return maxCapacity; + } + + public void setMaximumCapacity(String queue, float maxCapacity) { + if (maxCapacity > MAXIMUM_CAPACITY_VALUE) { + throw new IllegalArgumentException("Illegal " + + "maximum-capacity of " + maxCapacity + " for queue " + queue); + } + setFloat(getQueuePrefix(queue) + MAXIMUM_CAPACITY, maxCapacity); + LOG.debug("CSConf - setMaxCapacity: queuePrefix=" + getQueuePrefix(queue) + + ", maxCapacity=" + maxCapacity); + } + + public void setCapacityByLabel(String queue, String label, float capacity) { + setFloat(getNodeLabelPrefix(queue, label) + CAPACITY, capacity); + } + + public void setMaximumCapacityByLabel(String queue, String label, + float capacity) { + setFloat(getNodeLabelPrefix(queue, label) + MAXIMUM_CAPACITY, capacity); + } + + public int getUserLimit(String queue) { + int userLimit = getInt(getQueuePrefix(queue) + USER_LIMIT, + DEFAULT_USER_LIMIT); + return userLimit; + } + + public void setUserLimit(String queue, int userLimit) { + setInt(getQueuePrefix(queue) + USER_LIMIT, userLimit); + LOG.debug("here setUserLimit: queuePrefix=" + getQueuePrefix(queue) + + ", userLimit=" + getUserLimit(queue)); + } + + public float getUserLimitFactor(String queue) { + float userLimitFactor = + getFloat(getQueuePrefix(queue) + USER_LIMIT_FACTOR, + DEFAULT_USER_LIMIT_FACTOR); + return userLimitFactor; + } + + public void setUserLimitFactor(String queue, float userLimitFactor) { + setFloat(getQueuePrefix(queue) + USER_LIMIT_FACTOR, userLimitFactor); + } + + public QueueState getState(String queue) { + String state = get(getQueuePrefix(queue) + STATE); + return (state != null) ? QueueState.valueOf(state + .toUpperCase(Locale.ENGLISH)) : QueueState.RUNNING; + } + + public void setAccessibleNodeLabels(String queue, Set labels) { + if (labels == null) { + return; + } + String str = StringUtils.join(",", labels); + set(getQueuePrefix(queue) + ACCESSIBLE_NODE_LABELS, str); + } + + public Set getAccessibleNodeLabels(String queue) { + String accessibleLabelStr = + get(getQueuePrefix(queue) + ACCESSIBLE_NODE_LABELS); + + return CSConfUtils.getAccessibleNodeLabels(queue, accessibleLabelStr); + } + + private float internalGetLabeledQueueCapacity(String queue, String label, + String suffix, float defaultValue) { + String capacityPropertyName = getNodeLabelPrefix(queue, label) + suffix; + float capacity = getFloat(capacityPropertyName, defaultValue); + return CSConfUtils.getAndCheckLabeledQueueCapacity(queue, label, capacity, + defaultValue); + } + + public float getLabeledQueueCapacity(String queue, String label) { + return internalGetLabeledQueueCapacity(queue, label, CAPACITY, 0f); + } + + public float getLabeledQueueMaximumCapacity(String queue, String label) { + return internalGetLabeledQueueCapacity(queue, label, MAXIMUM_CAPACITY, 100f); + } + + public String getDefaultNodeLabelExpression(String queue) { + return get(getQueuePrefix(queue) + DEFAULT_NODE_LABEL_EXPRESSION); + } + + public void setDefaultNodeLabelExpression(String queue, String exp) { + set(getQueuePrefix(queue) + DEFAULT_NODE_LABEL_EXPRESSION, exp); + } + + /* + * Returns whether we should continue to look at all heart beating nodes even + * after the reservation limit was hit. The node heart beating in could + * satisfy the request thus could be a better pick then waiting for the + * reservation to be fullfilled. This config is refreshable. + */ + public boolean getReservationContinueLook() { + return getBoolean(RESERVE_CONT_LOOK_ALL_NODES, + DEFAULT_RESERVE_CONT_LOOK_ALL_NODES); + } + + public AccessControlList getAcl(String queue, QueueACL acl) { + String queuePrefix = getQueuePrefix(queue); + // The root queue defaults to all access if not defined + // Sub queues inherit access if not defined + String defaultAcl = queue.equals(ROOT) ? ALL_ACL : NONE_ACL; + String aclString = get(queuePrefix + CSConfUtils.getAclKey(acl), defaultAcl); + return new AccessControlList(aclString); + } + + public void setAcl(String queue, QueueACL acl, String aclString) { + String queuePrefix = getQueuePrefix(queue); + set(queuePrefix + CSConfUtils.getAclKey(acl), aclString); + } + + public Map getAcls(String queue) { + Map acls = + new HashMap(); + for (QueueACL acl : QueueACL.values()) { + acls.put(SchedulerUtils.toAccessType(acl), getAcl(queue, acl)); + } + return acls; + } + + public void setAcls(String queue, Map acls) { + for (Map.Entry e : acls.entrySet()) { + setAcl(queue, e.getKey(), e.getValue().getAclString()); + } + } + + public String[] getQueues(String queue) { + LOG.debug("CSConf - getQueues called for: queuePrefix=" + getQueuePrefix(queue)); + String[] queues = getStrings(getQueuePrefix(queue) + QUEUES); + List trimmedQueueNames = new ArrayList(); + if (null != queues) { + for (String s : queues) { + trimmedQueueNames.add(s.trim()); + } + queues = trimmedQueueNames.toArray(new String[0]); + } + + LOG.debug("CSConf - getQueues: queuePrefix=" + getQueuePrefix(queue) + + ", queues=" + ((queues == null) ? "" : StringUtils.arrayToString(queues))); + return queues; + } + + public void setQueues(String queue, String[] subQueues) { + set(getQueuePrefix(queue) + QUEUES, StringUtils.arrayToString(subQueues)); + LOG.debug("CSConf - setQueues: qPrefix=" + getQueuePrefix(queue) + + ", queues=" + StringUtils.arrayToString(subQueues)); + } + + public Resource getMinimumAllocation() { + int minimumMemory = getInt( + YarnConfiguration.RM_SCHEDULER_MINIMUM_ALLOCATION_MB, + YarnConfiguration.DEFAULT_RM_SCHEDULER_MINIMUM_ALLOCATION_MB); + int minimumCores = getInt( + YarnConfiguration.RM_SCHEDULER_MINIMUM_ALLOCATION_VCORES, + YarnConfiguration.DEFAULT_RM_SCHEDULER_MINIMUM_ALLOCATION_VCORES); + return Resources.createResource(minimumMemory, minimumCores); + } + + public Resource getMaximumAllocation() { + int maximumMemory = getInt( + YarnConfiguration.RM_SCHEDULER_MAXIMUM_ALLOCATION_MB, + YarnConfiguration.DEFAULT_RM_SCHEDULER_MAXIMUM_ALLOCATION_MB); + int maximumCores = getInt( + YarnConfiguration.RM_SCHEDULER_MAXIMUM_ALLOCATION_VCORES, + YarnConfiguration.DEFAULT_RM_SCHEDULER_MAXIMUM_ALLOCATION_VCORES); + return Resources.createResource(maximumMemory, maximumCores); + } + + /** + * Get the per queue setting for the maximum limit to allocate to + * each container request. + * + * @param queue + * name of the queue + * @return setting specified per queue else falls back to the cluster setting + */ + public Resource getMaximumAllocationPerQueue(String queue) { + String queuePrefix = getQueuePrefix(queue); + int maxAllocationMbPerQueue = + getInt(queuePrefix + MAXIMUM_ALLOCATION_MB, (int) UNDEFINED); + int maxAllocationVcoresPerQueue = + getInt(queuePrefix + MAXIMUM_ALLOCATION_VCORES, (int) UNDEFINED); + Resource clusterMax = getMaximumAllocation(); + return CSConfUtils.getMaximumAllocationPerQueue(queue, + maxAllocationMbPerQueue, maxAllocationVcoresPerQueue, clusterMax); + } + + public boolean getEnableUserMetrics() { + return getBoolean(ENABLE_USER_METRICS, DEFAULT_ENABLE_USER_METRICS); + } + + public int getNodeLocalityDelay() { + int delay = getInt(NODE_LOCALITY_DELAY, DEFAULT_NODE_LOCALITY_DELAY); + return (delay == DEFAULT_NODE_LOCALITY_DELAY) ? 0 : delay; + } + + public ResourceCalculator getResourceCalculator() { + return ReflectionUtils.newInstance( + getClass( + RESOURCE_CALCULATOR_CLASS, + DEFAULT_RESOURCE_CALCULATOR_CLASS, + ResourceCalculator.class), + this); + } + + public void setResourceCalculator( + Class resourceCalculatorClass) { + setClass( + RESOURCE_CALCULATOR_CLASS, + resourceCalculatorClass, + ResourceCalculator.class); + } + + public boolean getScheduleAynschronously() { + return getBoolean(SCHEDULE_ASYNCHRONOUSLY_ENABLE, + DEFAULT_SCHEDULE_ASYNCHRONOUSLY_ENABLE); + } + + public void setScheduleAynschronously(boolean async) { + setBoolean(SCHEDULE_ASYNCHRONOUSLY_ENABLE, async); + } + + public boolean getOverrideWithQueueMappings() { + return getBoolean(ENABLE_QUEUE_MAPPING_OVERRIDE, + DEFAULT_ENABLE_QUEUE_MAPPING_OVERRIDE); + } + + + + /** + * Get user/group mappings to queues. + * + * @return user/groups mappings or null on illegal configs + */ + public List getQueueMappings() { + return CSConfUtils.getQueueMappings(get(QUEUE_MAPPING)); + } + + public boolean isReservable(String queue) { + boolean isReservable = + getBoolean(getQueuePrefix(queue) + IS_RESERVABLE, false); + return isReservable; + } + + public void setReservable(String queue, boolean isReservable) { + setBoolean(getQueuePrefix(queue) + IS_RESERVABLE, isReservable); + LOG.debug("here setReservableQueue: queuePrefix=" + getQueuePrefix(queue) + + ", isReservableQueue=" + isReservable(queue)); + } + + @Override + public long getReservationWindow(String queue) { + long reservationWindow = + getLong(getQueuePrefix(queue) + RESERVATION_WINDOW, + ReservationSchedulerConfiguration.DEFAULT_RESERVATION_WINDOW); + return reservationWindow; + } + + @Override + public float getAverageCapacity(String queue) { + float avgCapacity = + getFloat(getQueuePrefix(queue) + AVERAGE_CAPACITY, + MAXIMUM_CAPACITY_VALUE); + return avgCapacity; + } + + @Override + public float getInstantaneousMaxCapacity(String queue) { + float instMaxCapacity = + getFloat(getQueuePrefix(queue) + INSTANTANEOUS_MAX_CAPACITY, + MAXIMUM_CAPACITY_VALUE); + return instMaxCapacity; + } + + public void setInstantaneousMaxCapacity(String queue, float instMaxCapacity) { + setFloat(getQueuePrefix(queue) + INSTANTANEOUS_MAX_CAPACITY, + instMaxCapacity); + } + + public void setReservationWindow(String queue, long reservationWindow) { + setLong(getQueuePrefix(queue) + RESERVATION_WINDOW, reservationWindow); + } + + public void setAverageCapacity(String queue, float avgCapacity) { + setFloat(getQueuePrefix(queue) + AVERAGE_CAPACITY, avgCapacity); + } + + @Override + public String getReservationAdmissionPolicy(String queue) { + String reservationPolicy = + get(getQueuePrefix(queue) + RESERVATION_ADMISSION_POLICY, + DEFAULT_RESERVATION_ADMISSION_POLICY); + return reservationPolicy; + } + + public void setReservationAdmissionPolicy(String queue, + String reservationPolicy) { + set(getQueuePrefix(queue) + RESERVATION_ADMISSION_POLICY, reservationPolicy); + } + + @Override + public String getReservationAgent(String queue) { + String reservationAgent = + get(getQueuePrefix(queue) + RESERVATION_AGENT_NAME, + DEFAULT_RESERVATION_AGENT_NAME); + return reservationAgent; + } + + public void setReservationAgent(String queue, String reservationPolicy) { + set(getQueuePrefix(queue) + RESERVATION_AGENT_NAME, reservationPolicy); + } + + @Override + public boolean getShowReservationAsQueues(String queuePath) { + boolean showReservationAsQueues = + getBoolean(getQueuePrefix(queuePath) + + RESERVATION_SHOW_RESERVATION_AS_QUEUE, + ReservationSchedulerConfiguration.DEFAULT_SHOW_RESERVATIONS_AS_QUEUES); + return showReservationAsQueues; + } + + @Override + public String getReplanner(String queue) { + String replanner = + get(getQueuePrefix(queue) + RESERVATION_PLANNER_NAME, + DEFAULT_RESERVATION_PLANNER_NAME); + return replanner; + } + + @Override + public boolean getMoveOnExpiry(String queue) { + boolean killOnExpiry = + getBoolean(getQueuePrefix(queue) + RESERVATION_MOVE_ON_EXPIRY, + ReservationSchedulerConfiguration.DEFAULT_RESERVATION_MOVE_ON_EXPIRY); + return killOnExpiry; + } + + @Override + public long getEnforcementWindow(String queue) { + long enforcementWindow = + getLong(getQueuePrefix(queue) + RESERVATION_ENFORCEMENT_WINDOW, + ReservationSchedulerConfiguration.DEFAULT_RESERVATION_ENFORCEMENT_WINDOW); + return enforcementWindow; + } + + /** + * Sets the disable_preemption property in order to indicate + * whether or not container preemption will be disabled for the specified + * queue. + * + * @param queue queue path + * @param preemptionDisabled true if preemption is disabled on queue + */ + public void setPreemptionDisabled(String queue, boolean preemptionDisabled) { + setBoolean(getQueuePrefix(queue) + QUEUE_PREEMPTION_DISABLED, + preemptionDisabled); + } + + /** + * Indicates whether preemption is disabled on the specified queue. + * + * @param queue queue path to query + * @param defaultVal used as default if the disable_preemption + * is not set in the configuration + * @return true if preemption is disabled on queue, false otherwise + */ + public boolean getPreemptionDisabled(String queue, boolean defaultVal) { + boolean preemptionDisabled = + getBoolean(getQueuePrefix(queue) + QUEUE_PREEMPTION_DISABLED, + defaultVal); + return preemptionDisabled; + } + + @Override + public long getAsyncSchedulerIntervalMs() { + return getLong(ASYNC_SCHEDULER_INTERVAL, + DEFAULT_ASYNC_SCHEDULER_INTERVAL); + } + + @Override + public void setMaximumApplicationsPerQueue(String queue, int value) { + setInt(getQueuePrefix(queue) + MAXIMUM_APPLICATIONS, value); + } + + @Override + public void setMaximumApplicationMasterResourcePerQueuePercent(String queue, + float value) { + setFloat(getQueuePrefix(queue) + + MAXIMUM_APPLICATION_MASTERS_RESOURCE_PERCENT, value); + } +} \ No newline at end of file diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacityScheduler.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacityScheduler.java index 6b9d846..32d1753 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacityScheduler.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacityScheduler.java @@ -25,6 +25,7 @@ import java.util.Comparator; import java.util.EnumSet; import java.util.HashMap; +import java.util.HashSet; import java.util.List; import java.util.Map; import java.util.Map.Entry; @@ -33,7 +34,6 @@ import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.atomic.AtomicBoolean; import java.util.concurrent.atomic.AtomicInteger; -import java.util.HashSet; import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; @@ -56,6 +56,7 @@ import org.apache.hadoop.yarn.api.records.QueueACL; import org.apache.hadoop.yarn.api.records.QueueInfo; import org.apache.hadoop.yarn.api.records.QueueUserACLInfo; +import org.apache.hadoop.yarn.api.records.ReservationId; import org.apache.hadoop.yarn.api.records.Resource; import org.apache.hadoop.yarn.api.records.ResourceOption; import org.apache.hadoop.yarn.api.records.ResourceRequest; @@ -84,12 +85,14 @@ import org.apache.hadoop.yarn.server.resourcemanager.scheduler.AbstractYarnScheduler; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.Allocation; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.PreemptableResourceScheduler; +import org.apache.hadoop.yarn.server.resourcemanager.scheduler.Queue; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.QueueMetrics; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.QueueNotFoundException; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.SchedulerApplication; +import org.apache.hadoop.yarn.server.resourcemanager.scheduler.SchedulerDynamicEditException; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.SchedulerUtils; -import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacitySchedulerConfiguration.QueueMapping; -import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacitySchedulerConfiguration.QueueMapping.MappingType; +import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.QueueMapping.MappingType; +import org.apache.hadoop.yarn.server.resourcemanager.scheduler.common.QueueEntitlement; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.common.fica.FiCaSchedulerApp; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.common.fica.FiCaSchedulerNode; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.event.AppAddedSchedulerEvent; @@ -112,11 +115,6 @@ import com.google.common.annotations.VisibleForTesting; import com.google.common.base.Preconditions; -import org.apache.hadoop.yarn.api.records.ReservationId; -import org.apache.hadoop.yarn.server.resourcemanager.scheduler.common.QueueEntitlement; -import org.apache.hadoop.yarn.server.resourcemanager.scheduler.SchedulerDynamicEditException; -import org.apache.hadoop.yarn.server.resourcemanager.scheduler.Queue; - @LimitedPrivate("yarn") @Evolving @SuppressWarnings("unchecked") @@ -200,7 +198,7 @@ public Configuration getConf() { return yarnConf; } - private CapacitySchedulerConfiguration conf; + private CSPlainConfiguration csConf; private Configuration yarnConf; private Map queues = new ConcurrentHashMap(); @@ -218,10 +216,6 @@ public Configuration getConf() { * EXPERT */ private long asyncScheduleInterval; - private static final String ASYNC_SCHEDULER_INTERVAL = - CapacitySchedulerConfiguration.SCHEDULE_ASYNCHRONOUSLY_PREFIX - + ".scheduling-interval-ms"; - private static final long DEFAULT_ASYNC_SCHEDULER_INTERVAL = 5; private boolean overrideWithQueueMappings = false; private List mappings = null; @@ -248,7 +242,7 @@ public CSQueue getRootQueue() { @Override public CapacitySchedulerConfiguration getConfiguration() { - return conf; + return csConf; } @Override @@ -286,26 +280,33 @@ public synchronized RMContext getRMContext() { public synchronized void setRMContext(RMContext rmContext) { this.rmContext = rmContext; } + + // Originally, this method is placed in CapacitySchedulerConfiguration, it + // shouldn't belong to that place. + private boolean getUsePortForNodeName(Configuration configuration) { + return configuration.getBoolean( + YarnConfiguration.RM_SCHEDULER_INCLUDE_PORT_IN_NODE_NAME, + YarnConfiguration.DEFAULT_RM_SCHEDULER_USE_PORT_FOR_NODE_NAME); + } private synchronized void initScheduler(Configuration configuration) throws IOException { - this.conf = loadCapacitySchedulerConfiguration(configuration); - validateConf(this.conf); - this.minimumAllocation = this.conf.getMinimumAllocation(); - initMaximumResourceCapability(this.conf.getMaximumAllocation()); - this.calculator = this.conf.getResourceCalculator(); - this.usePortForNodeName = this.conf.getUsePortForNodeName(); + this.csConf = loadCapacitySchedulerConfiguration(configuration); + validateConf(csConf); + this.minimumAllocation = this.csConf.getMinimumAllocation(); + initMaximumResourceCapability(this.csConf.getMaximumAllocation()); + this.calculator = this.csConf.getResourceCalculator(); + this.usePortForNodeName = getUsePortForNodeName(csConf); this.applications = new ConcurrentHashMap>(); this.labelManager = rmContext.getNodeLabelManager(); - authorizer = YarnAuthorizationProvider.getInstance(yarnConf); - initializeQueues(this.conf); + authorizer = YarnAuthorizationProvider.getInstance(csConf); + initializeQueues(this.csConf); - scheduleAsynchronously = this.conf.getScheduleAynschronously(); - asyncScheduleInterval = - this.conf.getLong(ASYNC_SCHEDULER_INTERVAL, - DEFAULT_ASYNC_SCHEDULER_INTERVAL); + scheduleAsynchronously = this.csConf.getScheduleAynschronously(); + asyncScheduleInterval = this.csConf.getAsyncSchedulerIntervalMs(); + if (scheduleAsynchronously) { asyncSchedulerThread = new AsyncScheduleThread(this); } @@ -354,16 +355,16 @@ public void serviceStop() throws Exception { public synchronized void reinitialize(Configuration conf, RMContext rmContext) throws IOException { Configuration configuration = new Configuration(conf); - CapacitySchedulerConfiguration oldConf = this.conf; - this.conf = loadCapacitySchedulerConfiguration(configuration); - validateConf(this.conf); + CSPlainConfiguration oldConf = this.csConf; + this.csConf = loadCapacitySchedulerConfiguration(configuration); + validateConf(this.csConf); try { LOG.info("Re-initializing queues..."); - refreshMaximumAllocation(this.conf.getMaximumAllocation()); - reinitializeQueues(this.conf); + refreshMaximumAllocation(this.csConf.getMaximumAllocation()); + reinitializeQueues(this.csConf); } catch (Throwable t) { - this.conf = oldConf; - refreshMaximumAllocation(this.conf.getMaximumAllocation()); + this.csConf = oldConf; + refreshMaximumAllocation(this.csConf.getMaximumAllocation()); throw new IOException("Failed to re-init queues", t); } } @@ -432,7 +433,7 @@ public void suspendSchedule() { @Private public static final String ROOT_QUEUE = - CapacitySchedulerConfiguration.PREFIX + CapacitySchedulerConfiguration.ROOT; + CSPlainConfiguration.PREFIX + CSPlainConfiguration.ROOT; static class QueueHook { public CSQueue hook(CSQueue queue) { @@ -442,11 +443,11 @@ public CSQueue hook(CSQueue queue) { private static final QueueHook noop = new QueueHook(); private void initializeQueueMappings() throws IOException { - overrideWithQueueMappings = conf.getOverrideWithQueueMappings(); + overrideWithQueueMappings = csConf.getOverrideWithQueueMappings(); LOG.info("Initialized queue mappings, override: " + overrideWithQueueMappings); // Get new user/group mappings - List newMappings = conf.getQueueMappings(); + List newMappings = csConf.getQueueMappings(); //check if mappings refer to valid queues for (QueueMapping mapping : newMappings) { if (!mapping.queue.equals(CURRENT_USER_MAPPING) && @@ -462,16 +463,16 @@ private void initializeQueueMappings() throws IOException { mappings = newMappings; // initialize groups if mappings are present if (mappings.size() > 0) { - groups = new Groups(conf); + groups = new Groups(yarnConf); } } @Lock(CapacityScheduler.class) - private void initializeQueues(CapacitySchedulerConfiguration conf) + private void initializeQueues(CSPlainConfiguration csConf) throws IOException { root = - parseQueue(this, conf, null, CapacitySchedulerConfiguration.ROOT, + parseQueue(this, csConf, null, CSPlainConfiguration.ROOT, queues, queues, noop); labelManager.reinitializeQueueLabels(getQueueToLabels()); LOG.info("Initialized root queue " + root); @@ -480,7 +481,7 @@ private void initializeQueues(CapacitySchedulerConfiguration conf) } @Lock(CapacityScheduler.class) - private void reinitializeQueues(CapacitySchedulerConfiguration conf) + private void reinitializeQueues(CSPlainConfiguration conf) throws IOException { // Parse new queues Map newQueues = new HashMap(); @@ -1387,7 +1388,7 @@ public synchronized boolean checkAccess(UserGroupInformation callerUGI, return apps; } - private CapacitySchedulerConfiguration loadCapacitySchedulerConfiguration( + private CSPlainConfiguration loadCapacitySchedulerConfiguration( Configuration configuration) throws IOException { try { InputStream CSInputStream = @@ -1396,9 +1397,9 @@ private CapacitySchedulerConfiguration loadCapacitySchedulerConfiguration( YarnConfiguration.CS_CONFIGURATION_FILE); if (CSInputStream != null) { configuration.addResource(CSInputStream); - return new CapacitySchedulerConfiguration(configuration, false); + return new CSPlainConfiguration(configuration, false); } - return new CapacitySchedulerConfiguration(configuration, true); + return new CSPlainConfiguration(configuration, true); } catch (Exception e) { throw new IOException(e); } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacitySchedulerConfiguration.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacitySchedulerConfiguration.java index 5e6d3eb..33ee6eb 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacitySchedulerConfiguration.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacitySchedulerConfiguration.java @@ -1,89 +1,51 @@ /** -* Licensed to the Apache Software Foundation (ASF) under one -* or more contributor license agreements. See the NOTICE file -* distributed with this work for additional information -* regarding copyright ownership. The ASF licenses this file -* to you under the Apache License, Version 2.0 (the -* "License"); you may not use this file except in compliance -* with the License. You may obtain a copy of the License at -* -* http://www.apache.org/licenses/LICENSE-2.0 -* -* Unless required by applicable law or agreed to in writing, software -* distributed under the License is distributed on an "AS IS" BASIS, -* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -* See the License for the specific language governing permissions and -* limitations under the License. -*/ + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ package org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity; -import java.util.ArrayList; -import java.util.Collection; -import java.util.Collections; -import java.util.HashMap; -import java.util.HashSet; import java.util.List; -import java.util.Locale; import java.util.Map; import java.util.Set; -import java.util.StringTokenizer; -import org.apache.commons.logging.Log; -import org.apache.commons.logging.LogFactory; import org.apache.hadoop.classification.InterfaceAudience.Private; -import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.security.authorize.AccessControlList; -import org.apache.hadoop.util.ReflectionUtils; -import org.apache.hadoop.util.StringUtils; import org.apache.hadoop.yarn.api.records.QueueACL; import org.apache.hadoop.yarn.api.records.QueueState; import org.apache.hadoop.yarn.api.records.Resource; -import org.apache.hadoop.yarn.conf.YarnConfiguration; -import org.apache.hadoop.yarn.nodelabels.CommonNodeLabelsManager; import org.apache.hadoop.yarn.security.AccessType; -import org.apache.hadoop.yarn.server.resourcemanager.nodelabels.RMNodeLabelsManager; import org.apache.hadoop.yarn.server.resourcemanager.reservation.ReservationSchedulerConfiguration; -import org.apache.hadoop.yarn.server.resourcemanager.scheduler.SchedulerUtils; import org.apache.hadoop.yarn.util.resource.DefaultResourceCalculator; import org.apache.hadoop.yarn.util.resource.ResourceCalculator; -import org.apache.hadoop.yarn.util.resource.Resources; -import com.google.common.collect.ImmutableSet; - -public class CapacitySchedulerConfiguration extends ReservationSchedulerConfiguration { - - private static final Log LOG = - LogFactory.getLog(CapacitySchedulerConfiguration.class); - - private static final String CS_CONFIGURATION_FILE = "capacity-scheduler.xml"; - - @Private - public static final String PREFIX = "yarn.scheduler.capacity."; - +public interface CapacitySchedulerConfiguration extends + ReservationSchedulerConfiguration { @Private public static final String DOT = "."; @Private - public static final String MAXIMUM_APPLICATIONS_SUFFIX = + public static final String MAXIMUM_APPLICATIONS = "maximum-applications"; - - @Private - public static final String MAXIMUM_SYSTEM_APPLICATIONS = - PREFIX + MAXIMUM_APPLICATIONS_SUFFIX; - + @Private - public static final String MAXIMUM_AM_RESOURCE_SUFFIX = + public static final String MAXIMUM_AM_RESOURCE = "maximum-am-resource-percent"; @Private - public static final String MAXIMUM_APPLICATION_MASTERS_RESOURCE_PERCENT = - PREFIX + MAXIMUM_AM_RESOURCE_SUFFIX; - - @Private - public static final String QUEUES = "queues"; - - @Private public static final String CAPACITY = "capacity"; @Private @@ -104,9 +66,9 @@ @Private public static final String DEFAULT_NODE_LABEL_EXPRESSION = "default-node-label-expression"; - - public static final String RESERVE_CONT_LOOK_ALL_NODES = PREFIX - + "reservations-continue-look-all-nodes"; + + public static final String RESERVE_CONT_LOOK_ALL_NODES_SUFFIX = + "reservations-continue-look-all-nodes"; @Private public static final boolean DEFAULT_RESERVE_CONT_LOOK_ALL_NODES = true; @@ -149,13 +111,15 @@ @Private public static final String NONE_ACL = " "; - @Private public static final String ENABLE_USER_METRICS = - PREFIX +"user-metrics.enable"; - @Private public static final boolean DEFAULT_ENABLE_USER_METRICS = false; + @Private + public static final String ENABLE_USER_METRICS = "user-metrics.enable"; + + @Private + public static final boolean DEFAULT_ENABLE_USER_METRICS = false; /** ResourceComparator for scheduling. */ - @Private public static final String RESOURCE_CALCULATOR_CLASS = - PREFIX + "resource-calculator"; + @Private + public static final String RESOURCE_CALCULATOR_CLASS = "resource-calculator"; @Private public static final Class DEFAULT_RESOURCE_CALCULATOR_CLASS = DefaultResourceCalculator.class; @@ -164,15 +128,14 @@ public static final String ROOT = "root"; @Private - public static final String NODE_LOCALITY_DELAY = - PREFIX + "node-locality-delay"; + public static final String NODE_LOCALITY_DELAY = "node-locality-delay"; @Private public static final int DEFAULT_NODE_LOCALITY_DELAY = -1; @Private public static final String SCHEDULE_ASYNCHRONOUSLY_PREFIX = - PREFIX + "schedule-asynchronously"; + "schedule-asynchronously"; @Private public static final String SCHEDULE_ASYNCHRONOUSLY_ENABLE = @@ -182,7 +145,7 @@ public static final boolean DEFAULT_SCHEDULE_ASYNCHRONOUSLY_ENABLE = false; @Private - public static final String QUEUE_MAPPING = PREFIX + "queue-mappings"; + public static final String QUEUE_MAPPING = "queue-mappings"; @Private public static final String ENABLE_QUEUE_MAPPING_OVERRIDE = QUEUE_MAPPING + "-override.enable"; @@ -192,35 +155,6 @@ @Private public static final String QUEUE_PREEMPTION_DISABLED = "disable_preemption"; - - @Private - public static class QueueMapping { - - public enum MappingType { - - USER("u"), - GROUP("g"); - private final String type; - private MappingType(String type) { - this.type = type; - } - - public String toString() { - return type; - } - - }; - - MappingType type; - String source; - String queue; - - public QueueMapping(MappingType type, String source, String queue) { - this.type = type; - this.source = source; - this.queue = queue; - } - } @Private public static final String AVERAGE_CAPACITY = "average-capacity"; @@ -256,58 +190,26 @@ public QueueMapping(MappingType type, String source, String queue) { @Private public static final String RESERVATION_ENFORCEMENT_WINDOW = "reservation-enforcement-window"; + + @Private + public static final String ASYNC_SCHEDULER_INTERVAL = "scheduling-interval-ms"; + + @Private + public static final long DEFAULT_ASYNC_SCHEDULER_INTERVAL = 5; - public CapacitySchedulerConfiguration() { - this(new Configuration()); - } - - public CapacitySchedulerConfiguration(Configuration configuration) { - this(configuration, true); - } - - public CapacitySchedulerConfiguration(Configuration configuration, - boolean useLocalConfigurationProvider) { - super(configuration); - if (useLocalConfigurationProvider) { - addResource(CS_CONFIGURATION_FILE); - } - } - - static String getQueuePrefix(String queue) { - String queueName = PREFIX + queue + DOT; - return queueName; - } - - private String getNodeLabelPrefix(String queue, String label) { - if (label.equals(CommonNodeLabelsManager.NO_LABEL)) { - return getQueuePrefix(queue); - } - return getQueuePrefix(queue) + ACCESSIBLE_NODE_LABELS + DOT + label + DOT; - } - - public int getMaximumSystemApplications() { - int maxApplications = - getInt(MAXIMUM_SYSTEM_APPLICATIONS, DEFAULT_MAXIMUM_SYSTEM_APPLICATIIONS); - return maxApplications; - } - - public float getMaximumApplicationMasterResourcePercent() { - return getFloat(MAXIMUM_APPLICATION_MASTERS_RESOURCE_PERCENT, - DEFAULT_MAXIMUM_APPLICATIONMASTERS_RESOURCE_PERCENT); - } - + + public int getMaximumSystemApplications(); + + public float getMaximumApplicationMasterResourcePercent(); /** * Get the maximum applications per queue setting. * @param queue name of the queue * @return setting specified or -1 if not set */ - public int getMaximumApplicationsPerQueue(String queue) { - int maxApplicationsPerQueue = - getInt(getQueuePrefix(queue) + MAXIMUM_APPLICATIONS_SUFFIX, - (int)UNDEFINED); - return maxApplicationsPerQueue; - } + public int getMaximumApplicationsPerQueue(String queue); + + public void setMaximumApplicationsPerQueue(String queue, int value); /** * Get the maximum am resource percent per queue setting. @@ -315,256 +217,63 @@ public int getMaximumApplicationsPerQueue(String queue) { * @return per queue setting or defaults to the global am-resource-percent * setting if per queue setting not present */ - public float getMaximumApplicationMasterResourcePerQueuePercent(String queue) { - return getFloat(getQueuePrefix(queue) + MAXIMUM_AM_RESOURCE_SUFFIX, - getMaximumApplicationMasterResourcePercent()); - } - - public float getNonLabeledQueueCapacity(String queue) { - float capacity = queue.equals("root") ? 100.0f : getFloat( - getQueuePrefix(queue) + CAPACITY, UNDEFINED); - if (capacity < MINIMUM_CAPACITY_VALUE || capacity > MAXIMUM_CAPACITY_VALUE) { - throw new IllegalArgumentException("Illegal " + - "capacity of " + capacity + " for queue " + queue); - } - LOG.debug("CSConf - getCapacity: queuePrefix=" + getQueuePrefix(queue) + - ", capacity=" + capacity); - return capacity; - } - - public void setCapacity(String queue, float capacity) { - if (queue.equals("root")) { - throw new IllegalArgumentException( - "Cannot set capacity, root queue has a fixed capacity of 100.0f"); - } - setFloat(getQueuePrefix(queue) + CAPACITY, capacity); - LOG.debug("CSConf - setCapacity: queuePrefix=" + getQueuePrefix(queue) + - ", capacity=" + capacity); - } - - public float getNonLabeledQueueMaximumCapacity(String queue) { - float maxCapacity = getFloat(getQueuePrefix(queue) + MAXIMUM_CAPACITY, - MAXIMUM_CAPACITY_VALUE); - maxCapacity = (maxCapacity == DEFAULT_MAXIMUM_CAPACITY_VALUE) ? - MAXIMUM_CAPACITY_VALUE : maxCapacity; - return maxCapacity; - } - - public void setMaximumCapacity(String queue, float maxCapacity) { - if (maxCapacity > MAXIMUM_CAPACITY_VALUE) { - throw new IllegalArgumentException("Illegal " + - "maximum-capacity of " + maxCapacity + " for queue " + queue); - } - setFloat(getQueuePrefix(queue) + MAXIMUM_CAPACITY, maxCapacity); - LOG.debug("CSConf - setMaxCapacity: queuePrefix=" + getQueuePrefix(queue) + - ", maxCapacity=" + maxCapacity); - } - - public void setCapacityByLabel(String queue, String label, float capacity) { - setFloat(getNodeLabelPrefix(queue, label) + CAPACITY, capacity); - } + public float getMaximumApplicationMasterResourcePerQueuePercent(String queue); + + public void setMaximumApplicationMasterResourcePerQueuePercent(String queue, + float value); + + public float getNonLabeledQueueCapacity(String queue); + + public void setCapacity(String queue, float capacity); + + public float getNonLabeledQueueMaximumCapacity(String queue); + + public void setMaximumCapacity(String queue, float maxCapacity); + + public void setCapacityByLabel(String queue, String label, float capacity); public void setMaximumCapacityByLabel(String queue, String label, - float capacity) { - setFloat(getNodeLabelPrefix(queue, label) + MAXIMUM_CAPACITY, capacity); - } - - public int getUserLimit(String queue) { - int userLimit = getInt(getQueuePrefix(queue) + USER_LIMIT, - DEFAULT_USER_LIMIT); - return userLimit; - } - - public void setUserLimit(String queue, int userLimit) { - setInt(getQueuePrefix(queue) + USER_LIMIT, userLimit); - LOG.debug("here setUserLimit: queuePrefix=" + getQueuePrefix(queue) + - ", userLimit=" + getUserLimit(queue)); - } - - public float getUserLimitFactor(String queue) { - float userLimitFactor = - getFloat(getQueuePrefix(queue) + USER_LIMIT_FACTOR, - DEFAULT_USER_LIMIT_FACTOR); - return userLimitFactor; - } - - public void setUserLimitFactor(String queue, float userLimitFactor) { - setFloat(getQueuePrefix(queue) + USER_LIMIT_FACTOR, userLimitFactor); - } - - public QueueState getState(String queue) { - String state = get(getQueuePrefix(queue) + STATE); - return (state != null) ? - QueueState.valueOf(state.toUpperCase(Locale.ENGLISH)) : - QueueState.RUNNING; - } - - public void setAccessibleNodeLabels(String queue, Set labels) { - if (labels == null) { - return; - } - String str = StringUtils.join(",", labels); - set(getQueuePrefix(queue) + ACCESSIBLE_NODE_LABELS, str); - } - - public Set getAccessibleNodeLabels(String queue) { - String accessibleLabelStr = - get(getQueuePrefix(queue) + ACCESSIBLE_NODE_LABELS); - - // When accessible-label is null, - if (accessibleLabelStr == null) { - // Only return null when queue is not ROOT - if (!queue.equals(ROOT)) { - return null; - } - } else { - // print a warning when accessibleNodeLabel specified in config and queue - // is ROOT - if (queue.equals(ROOT)) { - LOG.warn("Accessible node labels for root queue will be ignored," - + " it will be automatically set to \"*\"."); - } - } - - // always return ANY for queue root - if (queue.equals(ROOT)) { - return ImmutableSet.of(RMNodeLabelsManager.ANY); - } - - // In other cases, split the accessibleLabelStr by "," - Set set = new HashSet(); - for (String str : accessibleLabelStr.split(",")) { - if (!str.trim().isEmpty()) { - set.add(str.trim()); - } - } - - // if labels contains "*", only keep ANY behind - if (set.contains(RMNodeLabelsManager.ANY)) { - set.clear(); - set.add(RMNodeLabelsManager.ANY); - } - return Collections.unmodifiableSet(set); - } - - private float internalGetLabeledQueueCapacity(String queue, String label, String suffix, - float defaultValue) { - String capacityPropertyName = getNodeLabelPrefix(queue, label) + suffix; - float capacity = getFloat(capacityPropertyName, defaultValue); - if (capacity < MINIMUM_CAPACITY_VALUE - || capacity > MAXIMUM_CAPACITY_VALUE) { - throw new IllegalArgumentException("Illegal capacity of " + capacity - + " for node-label=" + label + " in queue=" + queue - + ", valid capacity should in range of [0, 100]."); - } - if (LOG.isDebugEnabled()) { - LOG.debug("CSConf - getCapacityOfLabel: prefix=" - + getNodeLabelPrefix(queue, label) + ", capacity=" + capacity); - } - return capacity; - } - - public float getLabeledQueueCapacity(String queue, String label) { - return internalGetLabeledQueueCapacity(queue, label, CAPACITY, 0f); - } - - public float getLabeledQueueMaximumCapacity(String queue, String label) { - return internalGetLabeledQueueCapacity(queue, label, MAXIMUM_CAPACITY, 100f); - } - - public String getDefaultNodeLabelExpression(String queue) { - return get(getQueuePrefix(queue) + DEFAULT_NODE_LABEL_EXPRESSION); - } - - public void setDefaultNodeLabelExpression(String queue, String exp) { - set(getQueuePrefix(queue) + DEFAULT_NODE_LABEL_EXPRESSION, exp); - } - - /* - * Returns whether we should continue to look at all heart beating nodes even - * after the reservation limit was hit. The node heart beating in could - * satisfy the request thus could be a better pick then waiting for the - * reservation to be fullfilled. This config is refreshable. - */ - public boolean getReservationContinueLook() { - return getBoolean(RESERVE_CONT_LOOK_ALL_NODES, - DEFAULT_RESERVE_CONT_LOOK_ALL_NODES); - } - - private static String getAclKey(QueueACL acl) { - return "acl_" + acl.toString().toLowerCase(Locale.ENGLISH); - } - - public AccessControlList getAcl(String queue, QueueACL acl) { - String queuePrefix = getQueuePrefix(queue); - // The root queue defaults to all access if not defined - // Sub queues inherit access if not defined - String defaultAcl = queue.equals(ROOT) ? ALL_ACL : NONE_ACL; - String aclString = get(queuePrefix + getAclKey(acl), defaultAcl); - return new AccessControlList(aclString); - } - - public void setAcl(String queue, QueueACL acl, String aclString) { - String queuePrefix = getQueuePrefix(queue); - set(queuePrefix + getAclKey(acl), aclString); - } - - public Map getAcls(String queue) { - Map acls = - new HashMap(); - for (QueueACL acl : QueueACL.values()) { - acls.put(SchedulerUtils.toAccessType(acl), getAcl(queue, acl)); - } - return acls; - } - - public void setAcls(String queue, Map acls) { - for (Map.Entry e : acls.entrySet()) { - setAcl(queue, e.getKey(), e.getValue().getAclString()); - } - } - - public String[] getQueues(String queue) { - LOG.debug("CSConf - getQueues called for: queuePrefix=" + getQueuePrefix(queue)); - String[] queues = getStrings(getQueuePrefix(queue) + QUEUES); - List trimmedQueueNames = new ArrayList(); - if (null != queues) { - for (String s : queues) { - trimmedQueueNames.add(s.trim()); - } - queues = trimmedQueueNames.toArray(new String[0]); - } - - LOG.debug("CSConf - getQueues: queuePrefix=" + getQueuePrefix(queue) + - ", queues=" + ((queues == null) ? "" : StringUtils.arrayToString(queues))); - return queues; - } - - public void setQueues(String queue, String[] subQueues) { - set(getQueuePrefix(queue) + QUEUES, StringUtils.arrayToString(subQueues)); - LOG.debug("CSConf - setQueues: qPrefix=" + getQueuePrefix(queue) + - ", queues=" + StringUtils.arrayToString(subQueues)); - } - - public Resource getMinimumAllocation() { - int minimumMemory = getInt( - YarnConfiguration.RM_SCHEDULER_MINIMUM_ALLOCATION_MB, - YarnConfiguration.DEFAULT_RM_SCHEDULER_MINIMUM_ALLOCATION_MB); - int minimumCores = getInt( - YarnConfiguration.RM_SCHEDULER_MINIMUM_ALLOCATION_VCORES, - YarnConfiguration.DEFAULT_RM_SCHEDULER_MINIMUM_ALLOCATION_VCORES); - return Resources.createResource(minimumMemory, minimumCores); - } - - public Resource getMaximumAllocation() { - int maximumMemory = getInt( - YarnConfiguration.RM_SCHEDULER_MAXIMUM_ALLOCATION_MB, - YarnConfiguration.DEFAULT_RM_SCHEDULER_MAXIMUM_ALLOCATION_MB); - int maximumCores = getInt( - YarnConfiguration.RM_SCHEDULER_MAXIMUM_ALLOCATION_VCORES, - YarnConfiguration.DEFAULT_RM_SCHEDULER_MAXIMUM_ALLOCATION_VCORES); - return Resources.createResource(maximumMemory, maximumCores); - } + float capacity); + + public int getUserLimit(String queue); + + public void setUserLimit(String queue, int userLimit); + + public float getUserLimitFactor(String queue); + + public void setUserLimitFactor(String queue, float userLimitFactor); + + public QueueState getState(String queue); + + public void setAccessibleNodeLabels(String queue, Set labels); + + public Set getAccessibleNodeLabels(String queue); + + public float getLabeledQueueCapacity(String queue, String label); + + public float getLabeledQueueMaximumCapacity(String queue, String label); + + public String getDefaultNodeLabelExpression(String queue); + + public void setDefaultNodeLabelExpression(String queue, String exp); + + public boolean getReservationContinueLook(); + + public AccessControlList getAcl(String queue, QueueACL acl); + + public void setAcl(String queue, QueueACL acl, String aclString); + + public Map getAcls(String queue); + + public void setAcls(String queue, Map acls); + + public String[] getQueues(String queue); + + public void setQueues(String queue, String[] subQueues); + + public Resource getMinimumAllocation(); + + public Resource getMaximumAllocation(); /** * Get the per queue setting for the maximum limit to allocate to @@ -574,266 +283,63 @@ public Resource getMaximumAllocation() { * name of the queue * @return setting specified per queue else falls back to the cluster setting */ - public Resource getMaximumAllocationPerQueue(String queue) { - String queuePrefix = getQueuePrefix(queue); - int maxAllocationMbPerQueue = getInt(queuePrefix + MAXIMUM_ALLOCATION_MB, - (int)UNDEFINED); - int maxAllocationVcoresPerQueue = getInt( - queuePrefix + MAXIMUM_ALLOCATION_VCORES, (int)UNDEFINED); - if (LOG.isDebugEnabled()) { - LOG.debug("max alloc mb per queue for " + queue + " is " - + maxAllocationMbPerQueue); - LOG.debug("max alloc vcores per queue for " + queue + " is " - + maxAllocationVcoresPerQueue); - } - Resource clusterMax = getMaximumAllocation(); - if (maxAllocationMbPerQueue == (int)UNDEFINED) { - LOG.info("max alloc mb per queue for " + queue + " is undefined"); - maxAllocationMbPerQueue = clusterMax.getMemory(); - } - if (maxAllocationVcoresPerQueue == (int)UNDEFINED) { - LOG.info("max alloc vcore per queue for " + queue + " is undefined"); - maxAllocationVcoresPerQueue = clusterMax.getVirtualCores(); - } - Resource result = Resources.createResource(maxAllocationMbPerQueue, - maxAllocationVcoresPerQueue); - if (maxAllocationMbPerQueue > clusterMax.getMemory() - || maxAllocationVcoresPerQueue > clusterMax.getVirtualCores()) { - throw new IllegalArgumentException( - "Queue maximum allocation cannot be larger than the cluster setting" - + " for queue " + queue - + " max allocation per queue: " + result - + " cluster setting: " + clusterMax); - } - return result; - } - - public boolean getEnableUserMetrics() { - return getBoolean(ENABLE_USER_METRICS, DEFAULT_ENABLE_USER_METRICS); - } - - public int getNodeLocalityDelay() { - int delay = getInt(NODE_LOCALITY_DELAY, DEFAULT_NODE_LOCALITY_DELAY); - return (delay == DEFAULT_NODE_LOCALITY_DELAY) ? 0 : delay; - } - - public ResourceCalculator getResourceCalculator() { - return ReflectionUtils.newInstance( - getClass( - RESOURCE_CALCULATOR_CLASS, - DEFAULT_RESOURCE_CALCULATOR_CLASS, - ResourceCalculator.class), - this); - } - - public boolean getUsePortForNodeName() { - return getBoolean(YarnConfiguration.RM_SCHEDULER_INCLUDE_PORT_IN_NODE_NAME, - YarnConfiguration.DEFAULT_RM_SCHEDULER_USE_PORT_FOR_NODE_NAME); - } - - public void setResourceComparator( - Class resourceCalculatorClass) { - setClass( - RESOURCE_CALCULATOR_CLASS, - resourceCalculatorClass, - ResourceCalculator.class); - } - - public boolean getScheduleAynschronously() { - return getBoolean(SCHEDULE_ASYNCHRONOUSLY_ENABLE, - DEFAULT_SCHEDULE_ASYNCHRONOUSLY_ENABLE); - } - - public void setScheduleAynschronously(boolean async) { - setBoolean(SCHEDULE_ASYNCHRONOUSLY_ENABLE, async); - } - - public boolean getOverrideWithQueueMappings() { - return getBoolean(ENABLE_QUEUE_MAPPING_OVERRIDE, - DEFAULT_ENABLE_QUEUE_MAPPING_OVERRIDE); - } + public Resource getMaximumAllocationPerQueue(String queue); - /** - * Returns a collection of strings, trimming leading and trailing whitespeace - * on each value - * - * @param str - * String to parse - * @param delim - * delimiter to separate the values - * @return Collection of parsed elements. - */ - private static Collection getTrimmedStringCollection(String str, - String delim) { - List values = new ArrayList(); - if (str == null) - return values; - StringTokenizer tokenizer = new StringTokenizer(str, delim); - while (tokenizer.hasMoreTokens()) { - String next = tokenizer.nextToken(); - if (next == null || next.trim().isEmpty()) { - continue; - } - values.add(next.trim()); - } - return values; - } + public boolean getEnableUserMetrics(); + + public int getNodeLocalityDelay(); + + public ResourceCalculator getResourceCalculator(); + + public void setResourceCalculator( + Class resourceCalculatorClass); + + public boolean getScheduleAynschronously(); + + public void setScheduleAynschronously(boolean async); + + public boolean getOverrideWithQueueMappings(); /** * Get user/group mappings to queues. * * @return user/groups mappings or null on illegal configs */ - public List getQueueMappings() { - List mappings = - new ArrayList(); - Collection mappingsString = - getTrimmedStringCollection(QUEUE_MAPPING); - for (String mappingValue : mappingsString) { - String[] mapping = - getTrimmedStringCollection(mappingValue, ":") - .toArray(new String[] {}); - if (mapping.length != 3 || mapping[1].length() == 0 - || mapping[2].length() == 0) { - throw new IllegalArgumentException( - "Illegal queue mapping " + mappingValue); - } - - QueueMapping m; - try { - QueueMapping.MappingType mappingType; - if (mapping[0].equals("u")) { - mappingType = QueueMapping.MappingType.USER; - } else if (mapping[0].equals("g")) { - mappingType = QueueMapping.MappingType.GROUP; - } else { - throw new IllegalArgumentException( - "unknown mapping prefix " + mapping[0]); - } - m = new QueueMapping( - mappingType, - mapping[1], - mapping[2]); - } catch (Throwable t) { - throw new IllegalArgumentException( - "Illegal queue mapping " + mappingValue); - } - - if (m != null) { - mappings.add(m); - } - } - - return mappings; - } - - public boolean isReservable(String queue) { - boolean isReservable = - getBoolean(getQueuePrefix(queue) + IS_RESERVABLE, false); - return isReservable; - } - - public void setReservable(String queue, boolean isReservable) { - setBoolean(getQueuePrefix(queue) + IS_RESERVABLE, isReservable); - LOG.debug("here setReservableQueue: queuePrefix=" + getQueuePrefix(queue) - + ", isReservableQueue=" + isReservable(queue)); - } - - @Override - public long getReservationWindow(String queue) { - long reservationWindow = - getLong(getQueuePrefix(queue) + RESERVATION_WINDOW, - DEFAULT_RESERVATION_WINDOW); - return reservationWindow; - } - - @Override - public float getAverageCapacity(String queue) { - float avgCapacity = - getFloat(getQueuePrefix(queue) + AVERAGE_CAPACITY, - MAXIMUM_CAPACITY_VALUE); - return avgCapacity; - } - - @Override - public float getInstantaneousMaxCapacity(String queue) { - float instMaxCapacity = - getFloat(getQueuePrefix(queue) + INSTANTANEOUS_MAX_CAPACITY, - MAXIMUM_CAPACITY_VALUE); - return instMaxCapacity; - } - - public void setInstantaneousMaxCapacity(String queue, float instMaxCapacity) { - setFloat(getQueuePrefix(queue) + INSTANTANEOUS_MAX_CAPACITY, - instMaxCapacity); - } - - public void setReservationWindow(String queue, long reservationWindow) { - setLong(getQueuePrefix(queue) + RESERVATION_WINDOW, reservationWindow); - } - - public void setAverageCapacity(String queue, float avgCapacity) { - setFloat(getQueuePrefix(queue) + AVERAGE_CAPACITY, avgCapacity); - } - - @Override - public String getReservationAdmissionPolicy(String queue) { - String reservationPolicy = - get(getQueuePrefix(queue) + RESERVATION_ADMISSION_POLICY, - DEFAULT_RESERVATION_ADMISSION_POLICY); - return reservationPolicy; - } + public List getQueueMappings(); + + public boolean isReservable(String queue); + + public void setReservable(String queue, boolean isReservable); + + public long getReservationWindow(String queue); + + public float getAverageCapacity(String queue); + + public float getInstantaneousMaxCapacity(String queue); + + public void setInstantaneousMaxCapacity(String queue, float instMaxCapacity); + + public void setReservationWindow(String queue, long reservationWindow); + + public void setAverageCapacity(String queue, float avgCapacity); + + public String getReservationAdmissionPolicy(String queue); public void setReservationAdmissionPolicy(String queue, - String reservationPolicy) { - set(getQueuePrefix(queue) + RESERVATION_ADMISSION_POLICY, reservationPolicy); - } - - @Override - public String getReservationAgent(String queue) { - String reservationAgent = - get(getQueuePrefix(queue) + RESERVATION_AGENT_NAME, - DEFAULT_RESERVATION_AGENT_NAME); - return reservationAgent; - } - - public void setReservationAgent(String queue, String reservationPolicy) { - set(getQueuePrefix(queue) + RESERVATION_AGENT_NAME, reservationPolicy); - } - - @Override - public boolean getShowReservationAsQueues(String queuePath) { - boolean showReservationAsQueues = - getBoolean(getQueuePrefix(queuePath) - + RESERVATION_SHOW_RESERVATION_AS_QUEUE, - DEFAULT_SHOW_RESERVATIONS_AS_QUEUES); - return showReservationAsQueues; - } - - @Override - public String getReplanner(String queue) { - String replanner = - get(getQueuePrefix(queue) + RESERVATION_PLANNER_NAME, - DEFAULT_RESERVATION_PLANNER_NAME); - return replanner; - } - - @Override - public boolean getMoveOnExpiry(String queue) { - boolean killOnExpiry = - getBoolean(getQueuePrefix(queue) + RESERVATION_MOVE_ON_EXPIRY, - DEFAULT_RESERVATION_MOVE_ON_EXPIRY); - return killOnExpiry; - } - - @Override - public long getEnforcementWindow(String queue) { - long enforcementWindow = - getLong(getQueuePrefix(queue) + RESERVATION_ENFORCEMENT_WINDOW, - DEFAULT_RESERVATION_ENFORCEMENT_WINDOW); - return enforcementWindow; - } + String reservationPolicy); + + public String getReservationAgent(String queue); + + public void setReservationAgent(String queue, String reservationPolicy); + public boolean getShowReservationAsQueues(String queuePath); + + public String getReplanner(String queue); + + public boolean getMoveOnExpiry(String queue); + + public long getEnforcementWindow(String queue); + /** * Sets the disable_preemption property in order to indicate * whether or not container preemption will be disabled for the specified @@ -842,10 +348,7 @@ public long getEnforcementWindow(String queue) { * @param queue queue path * @param preemptionDisabled true if preemption is disabled on queue */ - public void setPreemptionDisabled(String queue, boolean preemptionDisabled) { - setBoolean(getQueuePrefix(queue) + QUEUE_PREEMPTION_DISABLED, - preemptionDisabled); - } + public void setPreemptionDisabled(String queue, boolean preemptionDisabled); /** * Indicates whether preemption is disabled on the specified queue. @@ -855,10 +358,7 @@ public void setPreemptionDisabled(String queue, boolean preemptionDisabled) { * is not set in the configuration * @return true if preemption is disabled on queue, false otherwise */ - public boolean getPreemptionDisabled(String queue, boolean defaultVal) { - boolean preemptionDisabled = - getBoolean(getQueuePrefix(queue) + QUEUE_PREEMPTION_DISABLED, - defaultVal); - return preemptionDisabled; - } + public boolean getPreemptionDisabled(String queue, boolean defaultVal); + + public long getAsyncSchedulerIntervalMs(); } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/ParentQueue.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/ParentQueue.java index a26b0aa..235dedf 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/ParentQueue.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/ParentQueue.java @@ -90,10 +90,10 @@ public ParentQueue(CapacitySchedulerContext cs, float rawCapacity = cs.getConfiguration().getNonLabeledQueueCapacity(getQueuePath()); if (rootQueue && - (rawCapacity != CapacitySchedulerConfiguration.MAXIMUM_CAPACITY_VALUE)) { + (rawCapacity != CSPlainConfiguration.MAXIMUM_CAPACITY_VALUE)) { throw new IllegalArgumentException("Illegal " + "capacity of " + rawCapacity + " for queue " + queueName + - ". Must be " + CapacitySchedulerConfiguration.MAXIMUM_CAPACITY_VALUE); + ". Must be " + CSPlainConfiguration.MAXIMUM_CAPACITY_VALUE); } this.childQueues = new TreeSet(queueComparator); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/PlanQueue.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/PlanQueue.java index 7b53ad5..cd0b14b 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/PlanQueue.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/PlanQueue.java @@ -56,7 +56,7 @@ public PlanQueue(CapacitySchedulerContext cs, String queueName, showReservationsAsQueues = conf.getShowReservationAsQueues(queuePath); if (maxAppsForReservation < 0) { maxAppsForReservation = - (int) (CapacitySchedulerConfiguration.DEFAULT_MAXIMUM_SYSTEM_APPLICATIIONS * super + (int) (CSPlainConfiguration.DEFAULT_MAXIMUM_SYSTEM_APPLICATIIONS * super .getAbsoluteCapacity()); } int userLimit = conf.getUserLimit(queuePath); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/QueueMapping.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/QueueMapping.java new file mode 100644 index 0000000..a403903 --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/QueueMapping.java @@ -0,0 +1,47 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity; + +public class QueueMapping { + + public enum MappingType { + + USER("u"), + GROUP("g"); + private final String type; + private MappingType(String type) { + this.type = type; + } + + public String toString() { + return type; + } + + }; + + MappingType type; + String source; + String queue; + + public QueueMapping(MappingType type, String source, String queue) { + this.type = type; + this.source = source; + this.queue = queue; + } +} \ No newline at end of file diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/config/BaseSchedulerHierarchyConfiguration.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/config/BaseSchedulerHierarchyConfiguration.java new file mode 100644 index 0000000..e0ad8ce --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/config/BaseSchedulerHierarchyConfiguration.java @@ -0,0 +1,245 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.yarn.server.resourcemanager.scheduler.config; + +import java.util.Collection; +import java.util.Collections; +import java.util.HashMap; +import java.util.HashSet; +import java.util.List; +import java.util.Map; +import java.util.Set; + +import org.apache.hadoop.yarn.exceptions.YarnRuntimeException; + +import com.google.common.base.Function; +import com.google.common.collect.Collections2; +import com.google.common.collect.Sets; + +public class BaseSchedulerHierarchyConfiguration { + private static final String GLOBAL_TAG = "globals"; + private static final String QUEUES_TAG = "queues"; + private static final String QUEUE_TAG = "queue"; + private static final String ROOT_QUEUE_NAME = "root"; + private static final String DOT = "."; + + protected SchedulerConfNode root; + private Map pathToQueueNodes; + + public BaseSchedulerHierarchyConfiguration(SchedulerConfNode root) { + this.root = root; + pathToQueueNodes = new HashMap(); + + init(); + } + + public int getGlobalConfig(String optionName, int defaultValue) { + SchedulerConfNode global; + if (null == (global = root.getChild(GLOBAL_TAG))) { + return defaultValue; + } + return Integer.parseInt(global.getValue(optionName, + String.valueOf(defaultValue))); + } + + public float getGlobalConfig(String optionName, float defaultValue) { + SchedulerConfNode global; + if (null == (global = root.getChild(GLOBAL_TAG))) { + return defaultValue; + } + return Float.parseFloat(global.getValue(optionName, + String.valueOf(defaultValue))); + } + + public String getGlobalConfig(String optionName, String defaultValue) { + SchedulerConfNode global; + if (null == (global = root.getChild(GLOBAL_TAG))) { + return defaultValue; + } + return global.getValue(optionName, defaultValue); + } + + public boolean getGlobalConfig(String optionName, boolean defaultValue) { + SchedulerConfNode global; + if (null == (global = root.getChild(GLOBAL_TAG))) { + return defaultValue; + } + return Boolean.parseBoolean(global.getValue(optionName, + String.valueOf(defaultValue))); + } + + public long getGlobalConfig(String optionName, long defaultValue) { + SchedulerConfNode global; + if (null == (global = root.getChild(GLOBAL_TAG))) { + return defaultValue; + } + return Long.parseLong(global.getValue(optionName, + String.valueOf(defaultValue))); + } + + public void setGlobalConfig(String optionName, String value) { + SchedulerConfNode global; + if (null == (global = root.getChild(GLOBAL_TAG))) { + global = root.addChild(GLOBAL_TAG, "", ""); + } + global.setValue(optionName, value); + } + + protected void init() { + initQueues(); + } + + private void addQueues(SchedulerConfNode cur, String path) { + if (cur.getName().isEmpty()) { + throw new YarnRuntimeException("Queue's name couldn't be empty"); + } + + if (path.isEmpty()) { + path = cur.getName(); + } else { + path = path + DOT + cur.getName(); + } + pathToQueueNodes.put(path, cur); + + // add children + SchedulerConfNode queuesNode; + if (null != (queuesNode = cur.getChild(QUEUES_TAG))) { + List queues = queuesNode.getChildren(QUEUE_TAG); + // use a set to check if there are any duplicated queue names under the + // parent + Set names = new HashSet(); + for (SchedulerConfNode node : queues) { + if (!names.add(node.getName())) { + throw new YarnRuntimeException("duplicated queue has name=" + + node.getName() + " is already included by parent=" + + cur.getName()); + } + addQueues(node, path); + } + } + } + + private void initQueues() { + // Get root queue + SchedulerConfNode rootQueueNode = root.getChild(QUEUE_TAG); + if (null == rootQueueNode + || !rootQueueNode.getName().equals(ROOT_QUEUE_NAME)) { + throw new YarnRuntimeException("Failed to get root queue"); + } + + addQueues(rootQueueNode, ""); + } + + protected SchedulerConfNode getQueueNode(String queuePath) { + // Get queue conf node + SchedulerConfNode queueNode = pathToQueueNodes.get(queuePath); + if (null == queueNode) { + throw new YarnRuntimeException("Failed to get queue node with path=" + + queuePath); + } + return queueNode; + } + + public Collection getSubQueues(String queuePath) { + SchedulerConfNode queueNode = getQueueNode(queuePath); + SchedulerConfNode queuesNode = queueNode.getChild(QUEUES_TAG); + if (null == queuesNode) { + return Collections.emptyList(); + } + return queuesNode.getChildren(QUEUE_TAG); + } + + public Collection getSubQueueNames(String queuePath) { + Collection subQueueNames = + Collections2.transform(getSubQueues(queuePath), + new Function() { + @Override + public String apply(SchedulerConfNode input) { + return input.getName(); + } + }); + return subQueueNames; + } + + public void addQueues(String parentQueuePath, String[] children) { + SchedulerConfNode queueNode = getQueueNode(parentQueuePath); + SchedulerConfNode queuesNode = queueNode.getChild(QUEUES_TAG); + if (null == queuesNode) { + queueNode = queueNode.addChild(QUEUES_TAG, "", ""); + } + + // Check duplications + Set set = Sets.newHashSet(children); + if (set.size() < children.length || set.contains("")) { + throw new IllegalArgumentException( + "duplicated or empty named queue found in children"); + } + + for (String name : children) { + queueNode.addChild(QUEUE_TAG, name, ""); + } + } + + public int getQueueConfig(String queuePath, String optionName, + int defaultValue) { + return getQueueConfig(queuePath, optionName, defaultValue, false); + } + + public boolean getQueueConfig(String queuePath, String optionName, + boolean defaultValue) { + return Boolean.parseBoolean(getQueueConfig(queuePath, optionName, + String.valueOf(defaultValue), false)); + } + + public long getQueueConfig(String queuePath, String optionName, + long defaultValue) { + return Long.parseLong(getQueueConfig(queuePath, optionName, + String.valueOf(defaultValue), false)); + } + + public int getQueueConfig(String queuePath, String optionName, + int defaultValue, boolean inherit) { + SchedulerConfNode queueNode = getQueueNode(queuePath); + return Integer.parseInt(queueNode.getValue(optionName, + String.valueOf(defaultValue), inherit)); + } + + public float getQueueConfig(String queuePath, String optionName, + float defaultValue) { + return getQueueConfig(queuePath, optionName, defaultValue, false); + } + + public float getQueueConfig(String queuePath, String optionName, + float defaultValue, boolean inherit) { + SchedulerConfNode queueNode = getQueueNode(queuePath); + return Float.parseFloat(queueNode.getValue(optionName, + String.valueOf(defaultValue), inherit)); + } + + public String getQueueConfig(String queuePath, String optionName, + String defaultValue) { + return getQueueConfig(queuePath, optionName, defaultValue, false); + } + + public String getQueueConfig(String queuePath, String optionName, + String defaultValue, boolean inherit) { + SchedulerConfNode queueNode = getQueueNode(queuePath); + return queueNode.getValue(optionName, defaultValue, inherit); + } +} diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/config/SchedulerConfNode.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/config/SchedulerConfNode.java new file mode 100644 index 0000000..d484a66 --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/config/SchedulerConfNode.java @@ -0,0 +1,261 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.yarn.server.resourcemanager.scheduler.config; + +import java.util.Collections; +import java.util.List; + +import org.apache.commons.lang.StringUtils; + +import com.google.common.annotations.VisibleForTesting; +import com.google.common.collect.ArrayListMultimap; +import com.google.common.collect.ListMultimap; + +/** + * A "node" in scheduler's configuration file, it could be a queue, a user, etc. + * Generally, it should be a hierarchy + */ +public class SchedulerConfNode { + private static final String GLOBALS_TAGNAME = "globals"; + + private SchedulerConfNode parent; + private ListMultimap fields; + + /** + * Following fields in this class should be read only + */ + String name; + String type; + String value; + String tag; + + public SchedulerConfNode(String name, String type, String value, String tag, + ListMultimap fields) { + this.name = name; + this.type = type; + if (null == value) { + this.value = ""; + } else { + this.value = value; + } + this.tag = tag; + if (null == fields) { + this.fields = ArrayListMultimap.create(); + } else { + this.fields = fields; + } + parent = null; + } + + public String getName() { + return name; + } + + public String getType() { + return type; + } + + public String getTag() { + return tag; + } + + public void setParent(SchedulerConfNode parent) { + this.parent = parent; + } + + public SchedulerConfNode getParent() { + return this.parent; + } + + public void setFields(ListMultimap fields) { + if (fields == null) { + this.fields = ArrayListMultimap.create(); + } else { + this.fields = fields; + } + } + + /** + * Get children with given tag name, return null if no children related to + * given tagName + */ + public List getChildren(String tagName) { + if (null == fields.get(tagName) || fields.get(tagName).isEmpty()) { + return null; + } + return Collections.unmodifiableList(fields.get(tagName)); + } + + /** + * Get child with given tagName, if multiple children existed, return the + * first one, return null if no children existed. + */ + public SchedulerConfNode getChild(String tagName) { + List list = getChildren(tagName); + if (list == null || list.isEmpty()) { + return null; + } + return list.get(0); + } + + /** + * Get child with given tagName, if multiple children existed, return the + * first one, return null if no children existed. + */ + public SchedulerConfNode getChild(String tagName, String name) { + List list = getChildren(tagName); + if (list == null || list.isEmpty()) { + return null; + } + + for (SchedulerConfNode n : list) { + if (n.getName().equals(name)) { + return n; + } + } + + return null; + } + + /** + * Return value of this element itself. + * + * value + * + * Call getValue on now will return "value" + */ + public String getValue() { + return value; + } + + public String getValue(String tagName) { + return getValue(tagName, null); + } + + public String getValue(String tagName, String defaultValue) { + return getValue(tagName, defaultValue, false); + } + + /** + * Return value of direct child with given tagName, return empty string if no + * value; return null if no element with given tagName existed. + * + *
+   * {@code
+   * 
+   *  value1 
+   * 
+   * }
+   * 
+ * + * Call getValue("option1") will return value1. If no tag , + * defaultValue will be returned + * + * When inherit=true, it will try to find lowest ancestor with the option, + * such as: + * + *
+   * {@code
+   * 
+   *    value1
+   *    
+   *       value2
+   *       
+   *       
+   *    
+   * 
+   * }
+   * 
+ * + * Execute getValue("key", inherit=true) on "now" will returns value2 + */ + public String getValue(String tagName, String defaultValue, boolean inherit) { + // First to getValue on this node + SchedulerConfNode node = getChild(tagName); + if (null != node) { + return node.getValue(); + } + + if (inherit) { + // When inherit enabled: + if (null == parent) { + // We're root, try to get value from + SchedulerConfNode globals = getChild(GLOBALS_TAGNAME); + return globals.getValue(tagName, defaultValue, false); + } else { + // We're not root, try to inherit value from parent + return parent.getValue(tagName, defaultValue, inherit); + } + } else { + return defaultValue; + } + } + + public void setValue(String value) { + this.value = value; + } + + public void setValue(String tagName, String value) { + SchedulerConfNode node = getChild(tagName); + if (null != node) { + node.setValue(value); + } else { + fields.put(tagName, new SchedulerConfNode("", "", value, tagName, null)); + } + } + + public SchedulerConfNode addChild(String tagName, String name, String value) { + SchedulerConfNode child = new SchedulerConfNode(name, "", value, tagName, null); + fields.put(tagName, child); + return child; + } + + public boolean isLeafNode() { + return fields.isEmpty(); + } + + @VisibleForTesting + public ListMultimap getFields() { + return fields; + } + + private int getHierarchy() { + int hierarchy = 0; + SchedulerConfNode p = parent; + while (null != p) { + hierarchy ++; + p = p.parent; + } + return hierarchy; + } + + @Override + public String toString() { + StringBuilder sb = new StringBuilder(); + String indent = StringUtils.repeat(" ", getHierarchy() * 4); + sb.append(indent + "tag=" + tag + ", name=" + name + ", type=" + type + + ", value=" + value + " children=[\n"); + for (String key : fields.keySet()) { + for (SchedulerConfNode node : fields.get(key)) { + sb.append(node.toString()); + } + } + return sb.toString(); + } +} \ No newline at end of file diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/config/SchedulerConfParser.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/config/SchedulerConfParser.java new file mode 100644 index 0000000..654c2c3 --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/config/SchedulerConfParser.java @@ -0,0 +1,154 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.yarn.server.resourcemanager.scheduler.config; + +import java.io.ByteArrayInputStream; +import java.io.File; +import java.io.IOException; +import java.io.InputStream; +import java.util.ArrayList; +import java.util.List; + +import javax.xml.parsers.DocumentBuilder; +import javax.xml.parsers.DocumentBuilderFactory; +import javax.xml.parsers.ParserConfigurationException; + +import org.apache.hadoop.yarn.exceptions.YarnException; +import org.w3c.dom.Document; +import org.w3c.dom.Element; +import org.w3c.dom.Node; +import org.w3c.dom.NodeList; +import org.w3c.dom.Text; +import org.xml.sax.SAXException; + +import com.google.common.collect.ArrayListMultimap; +import com.google.common.collect.ListMultimap; + +/** + * Parse hierarchy scheduler configuration file + */ +public class SchedulerConfParser { + private static final String NAME = "name"; + private static final String TYPE = "type"; + + public static SchedulerConfNode parse(String content) throws YarnException { + try { + Document doc = parseDocument(content); + return internalParse(doc.getDocumentElement()); + } catch (ParserConfigurationException | SAXException | IOException e) { + throw new YarnException(e); + } + } + + public static SchedulerConfNode parse(InputStream is) throws YarnException { + try { + Document doc = parseDocument(is); + return internalParse(doc.getDocumentElement()); + } catch (ParserConfigurationException | SAXException | IOException e) { + throw new YarnException(e); + } + } + + public SchedulerConfNode parse(File file) throws YarnException { + try { + Document doc = parseDocument(file); + return internalParse(doc.getDocumentElement()); + } catch (ParserConfigurationException | SAXException | IOException e) { + throw new YarnException(e); + } + } + + private static SchedulerConfNode internalParse(Element current) { + // Get name + String name = current.getAttribute(NAME); + + // Get type + String type = current.getAttribute(TYPE); + + // Get tag + String tag = current.getTagName(); + + // Get value + String value = ""; + + // Initial fields + ListMultimap fields = ArrayListMultimap.create(); + List childrenElements = new ArrayList(); + NodeList nodes = current.getChildNodes(); + + if (nodes.getLength() > 0) { + for (int i = 0; i < nodes.getLength(); i++) { + Node node = nodes.item(i); + if (!(node instanceof Element)) { + continue; + } + Element element = (Element) node; + childrenElements.add(element); + SchedulerConfNode childConfNode = internalParse(element); + fields.put(element.getTagName(), childConfNode); + } + } + + if (childrenElements.isEmpty()) { + if (current.getFirstChild() != null) { + // Value will only be set when this element has no child + value = ((Text) current.getFirstChild()).getData().trim(); + } + } + + SchedulerConfNode now = + new SchedulerConfNode(name, type, value, tag, fields); + + // setParent for all children + for (SchedulerConfNode child : fields.values()) { + child.setParent(now); + } + + return now; + } + + private static DocumentBuilder getDocumentBuilder() + throws ParserConfigurationException { + DocumentBuilderFactory docBuilderFactor = + DocumentBuilderFactory.newInstance(); + docBuilderFactor.setIgnoringComments(true); + DocumentBuilder builder = docBuilderFactor.newDocumentBuilder(); + return builder; + } + + private static Document parseDocument(String content) + throws ParserConfigurationException, SAXException, IOException { + ByteArrayInputStream bais = new ByteArrayInputStream(content.getBytes()); + return parseDocument(bais); + } + + private static Document parseDocument(InputStream is) + throws ParserConfigurationException, SAXException, IOException { + DocumentBuilder builder = getDocumentBuilder(); + Document doc = builder.parse(is); + return doc; + } + + private static Document parseDocument(File file) + throws ParserConfigurationException, SAXException, IOException { + DocumentBuilder builder = getDocumentBuilder(); + Document doc = builder.parse(file); + return doc; + } +} diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/AllocationConfiguration.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/AllocationConfiguration.java index 9cb767d..f10c554 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/AllocationConfiguration.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/AllocationConfiguration.java @@ -36,7 +36,7 @@ import javax.annotation.concurrent.ThreadSafe; @ThreadSafe -public class AllocationConfiguration extends ReservationSchedulerConfiguration { +public class AllocationConfiguration implements ReservationSchedulerConfiguration { private static final AccessControlList EVERYBODY_ACL = new AccessControlList("*"); private static final AccessControlList NOBODY_ACL = new AccessControlList(" "); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestApplicationMasterService.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestApplicationMasterService.java index d1f0ede..fafd414 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestApplicationMasterService.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestApplicationMasterService.java @@ -20,16 +20,15 @@ import org.apache.hadoop.yarn.api.protocolrecords.RegisterApplicationMasterResponse; import org.apache.hadoop.yarn.proto.YarnServiceProtos.SchedulerResourceTypes; +import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CSPlainConfiguration; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacityScheduler; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacitySchedulerConfiguration; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.FairScheduler; import org.apache.hadoop.yarn.util.resource.DominantResourceCalculator; - import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; import org.apache.hadoop.yarn.api.protocolrecords.AllocateResponse; import org.apache.hadoop.yarn.api.protocolrecords.FinishApplicationMasterRequest; - import org.apache.hadoop.yarn.api.protocolrecords.impl.pb.AllocateRequestPBImpl; import org.apache.hadoop.yarn.api.records.*; import org.apache.hadoop.yarn.conf.YarnConfiguration; @@ -41,7 +40,6 @@ import org.apache.hadoop.yarn.server.resourcemanager.scheduler.ResourceScheduler; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fifo.FifoScheduler; import org.apache.hadoop.yarn.server.utils.BuilderUtils; - import org.junit.BeforeClass; import org.junit.Test; import org.junit.Assert; @@ -271,9 +269,9 @@ public void testResourceTypes() throws Exception { HashMap> driver = new HashMap>(); - CapacitySchedulerConfiguration csconf = - new CapacitySchedulerConfiguration(); - csconf.setResourceComparator(DominantResourceCalculator.class); + CSPlainConfiguration csconf = + new CSPlainConfiguration(); + csconf.setResourceCalculator(DominantResourceCalculator.class); YarnConfiguration testCapacityDRConf = new YarnConfiguration(csconf); testCapacityDRConf.setClass(YarnConfiguration.RM_SCHEDULER, CapacityScheduler.class, ResourceScheduler.class); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestClientRMService.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestClientRMService.java index dd2b3f8..4930628 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestClientRMService.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestClientRMService.java @@ -137,8 +137,8 @@ import org.apache.hadoop.yarn.server.resourcemanager.scheduler.ResourceScheduler; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.SchedulerAppReport; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.YarnScheduler; +import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CSPlainConfiguration; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacityScheduler; -import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacitySchedulerConfiguration; import org.apache.hadoop.yarn.server.resourcemanager.security.QueueACLsManager; import org.apache.hadoop.yarn.server.resourcemanager.security.RMDelegationTokenSecretManager; import org.apache.hadoop.yarn.server.security.ApplicationACLsManager; @@ -1295,7 +1295,7 @@ private static YarnScheduler mockYarnScheduler() { @Test public void testReservationAPIs() { // initialize - CapacitySchedulerConfiguration conf = new CapacitySchedulerConfiguration(); + CSPlainConfiguration conf = new CSPlainConfiguration(); ReservationSystemTestUtil.setupQueueConfiguration(conf); conf.setClass(YarnConfiguration.RM_SCHEDULER, CapacityScheduler.class, ResourceScheduler.class); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestFifoScheduler.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestFifoScheduler.java index 95aa856..b5f714a 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestFifoScheduler.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestFifoScheduler.java @@ -20,6 +20,7 @@ import static org.junit.Assert.assertTrue; import static org.junit.Assert.fail; +import static org.mockito.Mockito.mock; import java.util.ArrayList; import java.util.Collections; @@ -47,11 +48,10 @@ import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMApp; import org.apache.hadoop.yarn.server.resourcemanager.rmapp.attempt.RMAppAttempt; import org.apache.hadoop.yarn.server.resourcemanager.rmnode.RMNode; -import org.apache.hadoop.yarn.server.resourcemanager.scheduler.AbstractYarnScheduler; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.Allocation; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.ResourceScheduler; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.SchedulerNodeReport; -import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacitySchedulerConfiguration; +import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CSPlainConfiguration; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.event.AppAddedSchedulerEvent; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.event.AppAttemptAddedSchedulerEvent; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.event.NodeAddedSchedulerEvent; @@ -67,7 +67,6 @@ import org.junit.Assert; import org.junit.BeforeClass; import org.junit.Test; -import static org.mockito.Mockito.mock; public class TestFifoScheduler { private static final Log LOG = LogFactory.getLog(TestFifoScheduler.class); @@ -300,7 +299,7 @@ public void testNonDefaultMinimumAllocation() throws Exception { @Test (timeout = 50000) public void testReconnectedNode() throws Exception { - CapacitySchedulerConfiguration conf = new CapacitySchedulerConfiguration(); + CSPlainConfiguration conf = new CSPlainConfiguration(); conf.setQueues("default", new String[] {"default"}); conf.setCapacity("default", 100); FifoScheduler fs = new FifoScheduler(); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestRMAdminService.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestRMAdminService.java index 1bd3dbf..6f28ee1 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestRMAdminService.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestRMAdminService.java @@ -52,6 +52,7 @@ import org.apache.hadoop.yarn.server.api.protocolrecords.RefreshServiceAclsRequest; import org.apache.hadoop.yarn.server.api.protocolrecords.RefreshSuperUserGroupsConfigurationRequest; import org.apache.hadoop.yarn.server.api.protocolrecords.RefreshUserToGroupsMappingsRequest; +import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CSPlainConfiguration; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacityScheduler; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacitySchedulerConfiguration; import org.junit.After; @@ -145,8 +146,8 @@ public void testAdminRefreshQueuesWithFileSystemBasedConfigurationProvider() (CapacityScheduler) rm.getRMContext().getScheduler(); int maxAppsBefore = cs.getConfiguration().getMaximumSystemApplications(); - CapacitySchedulerConfiguration csConf = - new CapacitySchedulerConfiguration(); + CSPlainConfiguration csConf = + new CSPlainConfiguration(); csConf.set("yarn.scheduler.capacity.maximum-applications", "5000"); uploadConfiguration(csConf, "capacity-scheduler.xml"); @@ -553,8 +554,8 @@ public void testRMHAWithFileSystemBasedConfiguration() throws IOException, Assert.assertTrue(rm1.getRMContext().getHAServiceState() == HAServiceState.ACTIVE); - CapacitySchedulerConfiguration csConf = - new CapacitySchedulerConfiguration(); + CSPlainConfiguration csConf = + new CSPlainConfiguration(); csConf.set("yarn.scheduler.capacity.maximum-applications", "5000"); uploadConfiguration(csConf, "capacity-scheduler.xml"); @@ -641,8 +642,8 @@ public void testRMInitialsWithFileSystemBasedConfigurationProvider() + "/excludeHosts"); uploadConfiguration(yarnConf, "yarn-site.xml"); - CapacitySchedulerConfiguration csConf = - new CapacitySchedulerConfiguration(); + CSPlainConfiguration csConf = + new CSPlainConfiguration(); csConf.set("yarn.scheduler.capacity.maximum-applications", "5000"); uploadConfiguration(csConf, "capacity-scheduler.xml"); @@ -794,8 +795,8 @@ private void uploadDefaultConfiguration() throws IOException { "org.apache.hadoop.yarn.FileSystemBasedConfigurationProvider"); uploadConfiguration(yarnConf, "yarn-site.xml"); - CapacitySchedulerConfiguration csConf = - new CapacitySchedulerConfiguration(); + CSPlainConfiguration csConf = + new CSPlainConfiguration(); uploadConfiguration(csConf, "capacity-scheduler.xml"); Configuration hadoopPolicyConf = new Configuration(false); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestWorkPreservingRMRestart.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestWorkPreservingRMRestart.java index a9caf77..402d188 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestWorkPreservingRMRestart.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestWorkPreservingRMRestart.java @@ -66,6 +66,7 @@ import org.apache.hadoop.yarn.server.resourcemanager.scheduler.SchedulerApplication; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.SchedulerApplicationAttempt; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.SchedulerNode; +import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CSPlainConfiguration; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacityScheduler; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacitySchedulerConfiguration; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.LeafQueue; @@ -430,7 +431,7 @@ private void initFairScheduler(ResourceManager rm) throws IOException { private static final String USER_1 = "user1"; private static final String USER_2 = "user2"; - private void setupQueueConfiguration(CapacitySchedulerConfiguration conf) { + private void setupQueueConfiguration(CSPlainConfiguration conf) { conf.setQueues(CapacitySchedulerConfiguration.ROOT, new String[] { R }); final String Q_R = CapacitySchedulerConfiguration.ROOT + "." + R; conf.setCapacity(Q_R, 100); @@ -439,19 +440,19 @@ private void setupQueueConfiguration(CapacitySchedulerConfiguration conf) { conf.setQueues(Q_R, new String[] {A, B}); conf.setCapacity(Q_A, 50); conf.setCapacity(Q_B, 50); - conf.setDouble(CapacitySchedulerConfiguration + conf.setDouble(CSPlainConfiguration .MAXIMUM_APPLICATION_MASTERS_RESOURCE_PERCENT, 0.5f); } private void setupQueueConfigurationOnlyA( - CapacitySchedulerConfiguration conf) { + CSPlainConfiguration conf) { conf.setQueues(CapacitySchedulerConfiguration.ROOT, new String[] { R }); final String Q_R = CapacitySchedulerConfiguration.ROOT + "." + R; conf.setCapacity(Q_R, 100); final String Q_A = Q_R + "." + A; conf.setQueues(Q_R, new String[] {A}); conf.setCapacity(Q_A, 100); - conf.setDouble(CapacitySchedulerConfiguration + conf.setDouble(CSPlainConfiguration .MAXIMUM_APPLICATION_MASTERS_RESOURCE_PERCENT, 1.0f); } @@ -474,8 +475,8 @@ public void testCapacitySchedulerRecovery() throws Exception { conf.setBoolean(CapacitySchedulerConfiguration.ENABLE_USER_METRICS, true); conf.set(CapacitySchedulerConfiguration.RESOURCE_CALCULATOR_CLASS, DominantResourceCalculator.class.getName()); - CapacitySchedulerConfiguration csConf = - new CapacitySchedulerConfiguration(conf); + CSPlainConfiguration csConf = + new CSPlainConfiguration(conf); setupQueueConfiguration(csConf); MemoryRMStateStore memStore = new MemoryRMStateStore(); memStore.init(csConf); @@ -593,8 +594,8 @@ public void testCapacitySchedulerQueueRemovedRecovery() throws Exception { conf.setBoolean(CapacitySchedulerConfiguration.ENABLE_USER_METRICS, true); conf.set(CapacitySchedulerConfiguration.RESOURCE_CALCULATOR_CLASS, DominantResourceCalculator.class.getName()); - CapacitySchedulerConfiguration csConf = - new CapacitySchedulerConfiguration(conf); + CSPlainConfiguration csConf = + new CSPlainConfiguration(conf); setupQueueConfiguration(csConf); MemoryRMStateStore memStore = new MemoryRMStateStore(); memStore.init(csConf); @@ -625,8 +626,7 @@ public void testCapacitySchedulerQueueRemovedRecovery() throws Exception { rm1.clearQueueMetrics(app2); // Re-start RM - csConf = - new CapacitySchedulerConfiguration(conf); + csConf = new CSPlainConfiguration(conf); setupQueueConfigurationOnlyA(csConf); rm2 = new MockRM(csConf, memStore); rm2.start(); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/applicationsmanager/TestAMRMRPCNodeUpdates.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/applicationsmanager/TestAMRMRPCNodeUpdates.java index f4cb3b3..ec30147 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/applicationsmanager/TestAMRMRPCNodeUpdates.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/applicationsmanager/TestAMRMRPCNodeUpdates.java @@ -21,8 +21,6 @@ import java.security.PrivilegedExceptionAction; import java.util.List; -import org.junit.Assert; - import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.security.UserGroupInformation; import org.apache.hadoop.security.token.Token; @@ -41,9 +39,10 @@ import org.apache.hadoop.yarn.server.resourcemanager.MockRM; import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMApp; import org.apache.hadoop.yarn.server.resourcemanager.rmapp.attempt.RMAppAttempt; +import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CSPlainConfiguration; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.event.SchedulerEvent; -import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacitySchedulerConfiguration; import org.junit.After; +import org.junit.Assert; import org.junit.Before; import org.junit.Test; @@ -59,7 +58,7 @@ public void setUp() { @Override public void init(Configuration conf) { conf.set( - CapacitySchedulerConfiguration.MAXIMUM_APPLICATION_MASTERS_RESOURCE_PERCENT, + CSPlainConfiguration.MAXIMUM_APPLICATION_MASTERS_RESOURCE_PERCENT, "1.0"); super.init(conf); } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/monitor/capacity/TestProportionalCapacityPreemptionPolicy.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/monitor/capacity/TestProportionalCapacityPreemptionPolicy.java index 696b9bb..32254f3 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/monitor/capacity/TestProportionalCapacityPreemptionPolicy.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/monitor/capacity/TestProportionalCapacityPreemptionPolicy.java @@ -29,6 +29,8 @@ import static org.junit.Assert.assertNotNull; import static org.junit.Assert.assertTrue; import static org.junit.Assert.fail; +import static org.mockito.Matchers.any; +import static org.mockito.Matchers.anyString; import static org.mockito.Matchers.argThat; import static org.mockito.Matchers.isA; import static org.mockito.Mockito.mock; @@ -36,8 +38,6 @@ import static org.mockito.Mockito.times; import static org.mockito.Mockito.verify; import static org.mockito.Mockito.when; -import static org.mockito.Matchers.any; -import static org.mockito.Matchers.anyString; import java.util.ArrayList; import java.util.Comparator; @@ -53,7 +53,6 @@ import java.util.StringTokenizer; import java.util.TreeSet; -import org.apache.commons.collections.map.HashedMap; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.service.Service; import org.apache.hadoop.yarn.api.records.ApplicationAttemptId; @@ -72,6 +71,7 @@ import org.apache.hadoop.yarn.server.resourcemanager.rmcontainer.RMContainer; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.ContainerPreemptEvent; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.ContainerPreemptEventType; +import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CSPlainConfiguration; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CSQueue; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacityScheduler; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacitySchedulerConfiguration; @@ -88,7 +88,6 @@ import org.junit.rules.TestName; import org.mockito.ArgumentCaptor; import org.mockito.ArgumentMatcher; -import org.mortbay.log.Log; public class TestProportionalCapacityPreemptionPolicy { @@ -159,7 +158,7 @@ public void setup() { mCS = mock(CapacityScheduler.class); when(mCS.getResourceCalculator()).thenReturn(rc); lm = mock(RMNodeLabelsManager.class); - schedConf = new CapacitySchedulerConfiguration(); + schedConf = new CSPlainConfiguration(); when(mCS.getConfiguration()).thenReturn(schedConf); rmContext = mock(RMContext.class); when(mCS.getRMContext()).thenReturn(rmContext); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/reservation/ReservationSystemTestUtil.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/reservation/ReservationSystemTestUtil.java index bfaf06b..c2cb7b0 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/reservation/ReservationSystemTestUtil.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/reservation/ReservationSystemTestUtil.java @@ -47,6 +47,7 @@ import org.apache.hadoop.yarn.server.resourcemanager.nodelabels.RMNodeLabelsManager; import org.apache.hadoop.yarn.server.resourcemanager.rmnode.RMNode; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.AbstractYarnScheduler; +import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CSPlainConfiguration; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacityScheduler; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacitySchedulerConfiguration; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.event.NodeAddedSchedulerEvent; @@ -188,7 +189,7 @@ public static FairScheduler setupFairScheduler( public CapacityScheduler mockCapacityScheduler(int numContainers) throws IOException { // stolen from TestCapacityScheduler - CapacitySchedulerConfiguration conf = new CapacitySchedulerConfiguration(); + CSPlainConfiguration conf = new CSPlainConfiguration(); setupQueueConfiguration(conf); CapacityScheduler cs = Mockito.spy(new CapacityScheduler()); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/reservation/TestCapacityReservationSystem.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/reservation/TestCapacityReservationSystem.java index 11e52c4..7df75bb 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/reservation/TestCapacityReservationSystem.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/reservation/TestCapacityReservationSystem.java @@ -21,8 +21,8 @@ import org.apache.hadoop.yarn.exceptions.YarnException; import org.apache.hadoop.yarn.server.resourcemanager.RMContext; +import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CSPlainConfiguration; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacityScheduler; -import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacitySchedulerConfiguration; import org.junit.Assert; import org.junit.Test; @@ -62,11 +62,12 @@ public void testReinitialize() { } CapacityReservationSystem reservationSystem = new CapacityReservationSystem(); - CapacitySchedulerConfiguration conf = capScheduler.getConfiguration(); + CSPlainConfiguration conf = + (CSPlainConfiguration) capScheduler.getConfiguration(); RMContext mockContext = capScheduler.getRMContext(); reservationSystem.setRMContext(mockContext); try { - reservationSystem.reinitialize(capScheduler.getConfiguration(), + reservationSystem.reinitialize(conf, mockContext); } catch (YarnException e) { Assert.fail(e.getMessage()); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/reservation/TestCapacitySchedulerPlanFollower.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/reservation/TestCapacitySchedulerPlanFollower.java index b8663f6..1a0479f 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/reservation/TestCapacitySchedulerPlanFollower.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/reservation/TestCapacitySchedulerPlanFollower.java @@ -37,6 +37,7 @@ import org.apache.hadoop.yarn.server.resourcemanager.reservation.exceptions.PlanningException; import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMApp; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.Queue; +import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CSPlainConfiguration; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CSQueue; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacityScheduler; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacitySchedulerConfiguration; @@ -84,8 +85,8 @@ public void setUp() throws Exception { when(spyRMContext.getRMApps()).thenReturn(spyApps); when(spyRMContext.getScheduler()).thenReturn(scheduler); - CapacitySchedulerConfiguration csConf = - new CapacitySchedulerConfiguration(); + CSPlainConfiguration csConf = + new CSPlainConfiguration(); ReservationSystemTestUtil.setupQueueConfiguration(csConf); cs.setConf(csConf); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestApplicationLimits.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestApplicationLimits.java index 81a5aad..31f7f0b 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestApplicationLimits.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestApplicationLimits.java @@ -80,7 +80,7 @@ @Before public void setUp() throws IOException { CapacitySchedulerConfiguration csConf = - new CapacitySchedulerConfiguration(); + new CSPlainConfiguration(); YarnConfiguration conf = new YarnConfiguration(); setupQueueConfiguration(csConf); @@ -243,7 +243,7 @@ public void testAMResourceLimit() throws Exception { @Test public void testLimitsComputation() throws Exception { CapacitySchedulerConfiguration csConf = - new CapacitySchedulerConfiguration(); + new CSPlainConfiguration(); setupQueueConfiguration(csConf); YarnConfiguration conf = new YarnConfiguration(); @@ -322,11 +322,8 @@ public void testLimitsComputation() throws Exception { ); // Change the per-queue max AM resources percentage. - csConf.setFloat( - "yarn.scheduler.capacity." + - queue.getQueuePath() + - ".maximum-am-resource-percent", - 0.5f); + csConf.setMaximumApplicationMasterResourcePerQueuePercent( + queue.getQueuePath(), 0.5f); // Re-create queues to get new configs. queues = new HashMap(); root = @@ -346,10 +343,7 @@ public void testLimitsComputation() throws Exception { Resource.newInstance(400*GB, 1)); // Change the per-queue max applications. - csConf.setInt( - "yarn.scheduler.capacity." + - queue.getQueuePath() + - ".maximum-applications", 9999); + csConf.setMaximumApplicationsPerQueue(queue.getQueuePath(), 9999); // Re-create queues to get new configs. queues = new HashMap(); root = @@ -540,7 +534,7 @@ public void testActiveLimitsWithKilledApps() throws Exception { @Test public void testHeadroom() throws Exception { CapacitySchedulerConfiguration csConf = - new CapacitySchedulerConfiguration(); + new CSPlainConfiguration(); csConf.setUserLimit(CapacitySchedulerConfiguration.ROOT + "." + A, 25); setupQueueConfiguration(csConf); YarnConfiguration conf = new YarnConfiguration(); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCSQueueUtils.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCSQueueUtils.java index 5135ba9..eb1962e 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCSQueueUtils.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCSQueueUtils.java @@ -58,7 +58,7 @@ public void runInvalidDivisorTest(boolean useDominant) throws Exception { } YarnConfiguration conf = new YarnConfiguration(); - CapacitySchedulerConfiguration csConf = new CapacitySchedulerConfiguration(); + CapacitySchedulerConfiguration csConf = new CSPlainConfiguration(); CapacitySchedulerContext csContext = mock(CapacitySchedulerContext.class); when(csContext.getConf()).thenReturn(conf); @@ -101,7 +101,7 @@ public void testAbsoluteMaxAvailCapacityNoUse() throws Exception { Resource clusterResource = Resources.createResource(100 * 16 * GB, 100 * 32); YarnConfiguration conf = new YarnConfiguration(); - CapacitySchedulerConfiguration csConf = new CapacitySchedulerConfiguration(); + CapacitySchedulerConfiguration csConf = new CSPlainConfiguration(); CapacitySchedulerContext csContext = mock(CapacitySchedulerContext.class); when(csContext.getConf()).thenReturn(conf); @@ -147,7 +147,7 @@ public void testAbsoluteMaxAvailCapacityWithUse() throws Exception { Resource clusterResource = Resources.createResource(100 * 16 * GB, 100 * 32); YarnConfiguration conf = new YarnConfiguration(); - CapacitySchedulerConfiguration csConf = new CapacitySchedulerConfiguration(); + CapacitySchedulerConfiguration csConf = new CSPlainConfiguration(); CapacitySchedulerContext csContext = mock(CapacitySchedulerContext.class); when(csContext.getConf()).thenReturn(conf); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacityScheduler.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacityScheduler.java index fabf47d..6aac5d4 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacityScheduler.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacityScheduler.java @@ -132,8 +132,8 @@ private static final Log LOG = LogFactory.getLog(TestCapacityScheduler.class); private final int GB = 1024; - private static final String A = CapacitySchedulerConfiguration.ROOT + ".a"; - private static final String B = CapacitySchedulerConfiguration.ROOT + ".b"; + private static final String A = CSPlainConfiguration.ROOT + ".a"; + private static final String B = CSPlainConfiguration.ROOT + ".b"; private static final String A1 = A + ".a1"; private static final String A2 = A + ".a2"; private static final String B1 = B + ".b1"; @@ -160,8 +160,8 @@ protected RMNodeLabelsManager createNodeLabelManager() { return mgr; } }; - CapacitySchedulerConfiguration csConf - = new CapacitySchedulerConfiguration(); + CSPlainConfiguration csConf + = new CSPlainConfiguration(); setupQueueConfiguration(csConf); YarnConfiguration conf = new YarnConfiguration(csConf); conf.setClass(YarnConfiguration.RM_SCHEDULER, @@ -359,10 +359,10 @@ private void nodeUpdate( resourceManager.getResourceScheduler().handle(nodeUpdate); } - private void setupQueueConfiguration(CapacitySchedulerConfiguration conf) { + private void setupQueueConfiguration(CSPlainConfiguration conf) { // Define top-level queues - conf.setQueues(CapacitySchedulerConfiguration.ROOT, new String[] {"a", "b"}); + conf.setQueues(CSPlainConfiguration.ROOT, new String[] {"a", "b"}); conf.setCapacity(A, A_CAPACITY); conf.setCapacity(B, B_CAPACITY); @@ -388,19 +388,19 @@ private void setupQueueConfiguration(CapacitySchedulerConfiguration conf) { @Test public void testMaximumCapacitySetup() { float delta = 0.0000001f; - CapacitySchedulerConfiguration conf = new CapacitySchedulerConfiguration(); - assertEquals(CapacitySchedulerConfiguration.MAXIMUM_CAPACITY_VALUE,conf.getNonLabeledQueueMaximumCapacity(A),delta); + CSPlainConfiguration conf = new CSPlainConfiguration(); + assertEquals(CSPlainConfiguration.MAXIMUM_CAPACITY_VALUE,conf.getNonLabeledQueueMaximumCapacity(A),delta); conf.setMaximumCapacity(A, 50.0f); assertEquals(50.0f, conf.getNonLabeledQueueMaximumCapacity(A),delta); conf.setMaximumCapacity(A, -1); - assertEquals(CapacitySchedulerConfiguration.MAXIMUM_CAPACITY_VALUE,conf.getNonLabeledQueueMaximumCapacity(A),delta); + assertEquals(CSPlainConfiguration.MAXIMUM_CAPACITY_VALUE,conf.getNonLabeledQueueMaximumCapacity(A),delta); } @Test public void testRefreshQueues() throws Exception { CapacityScheduler cs = new CapacityScheduler(); - CapacitySchedulerConfiguration conf = new CapacitySchedulerConfiguration(); + CSPlainConfiguration conf = new CSPlainConfiguration(); RMContextImpl rmContext = new RMContextImpl(null, null, null, null, null, null, new RMContainerTokenSecretManager(conf), new NMTokenSecretManagerInRM(conf), @@ -501,14 +501,14 @@ public void testParseQueue() throws IOException { CapacityScheduler cs = new CapacityScheduler(); cs.setConf(new YarnConfiguration()); cs.setRMContext(resourceManager.getRMContext()); - CapacitySchedulerConfiguration conf = new CapacitySchedulerConfiguration(); + CSPlainConfiguration conf = new CSPlainConfiguration(); setupQueueConfiguration(conf); cs.init(conf); cs.start(); - conf.setQueues(CapacitySchedulerConfiguration.ROOT + ".a.a1", new String[] {"b1"} ); - conf.setCapacity(CapacitySchedulerConfiguration.ROOT + ".a.a1.b1", 100.0f); - conf.setUserLimitFactor(CapacitySchedulerConfiguration.ROOT + ".a.a1.b1", 100.0f); + conf.setQueues(CSPlainConfiguration.ROOT + ".a.a1", new String[] {"b1"} ); + conf.setCapacity(CSPlainConfiguration.ROOT + ".a.a1.b1", 100.0f); + conf.setUserLimitFactor(CSPlainConfiguration.ROOT + ".a.a1.b1", 100.0f); cs.reinitialize(conf, new RMContextImpl(null, null, null, null, null, null, new RMContainerTokenSecretManager(conf), @@ -518,8 +518,8 @@ public void testParseQueue() throws IOException { @Test public void testReconnectedNode() throws Exception { - CapacitySchedulerConfiguration csConf = - new CapacitySchedulerConfiguration(); + CSPlainConfiguration csConf = + new CSPlainConfiguration(); setupQueueConfiguration(csConf); CapacityScheduler cs = new CapacityScheduler(); cs.setConf(new YarnConfiguration()); @@ -551,7 +551,7 @@ public void testReconnectedNode() throws Exception { @Test public void testRefreshQueuesWithNewQueue() throws Exception { CapacityScheduler cs = new CapacityScheduler(); - CapacitySchedulerConfiguration conf = new CapacitySchedulerConfiguration(); + CSPlainConfiguration conf = new CSPlainConfiguration(); setupQueueConfiguration(conf); cs.setConf(new YarnConfiguration()); cs.setRMContext(resourceManager.getRMContext()); @@ -801,7 +801,7 @@ public void testGetAppsInQueue() throws Exception { @Test public void testAddAndRemoveAppFromCapacityScheduler() throws Exception { - CapacitySchedulerConfiguration conf = new CapacitySchedulerConfiguration(); + CSPlainConfiguration conf = new CSPlainConfiguration(); setupQueueConfiguration(conf); conf.setClass(YarnConfiguration.RM_SCHEDULER, CapacityScheduler.class, ResourceScheduler.class); @@ -977,8 +977,8 @@ public void testNumClusterNodes() throws Exception { cs.setConf(conf); RMContext rmContext = TestUtils.getMockRMContext(); cs.setRMContext(rmContext); - CapacitySchedulerConfiguration csConf = - new CapacitySchedulerConfiguration(); + CSPlainConfiguration csConf = + new CSPlainConfiguration(); setupQueueConfiguration(csConf); cs.init(csConf); cs.start(); @@ -1139,7 +1139,7 @@ public void testRecoverRequestAfterPreemption() throws Exception { } private MockRM setUpMove() { - CapacitySchedulerConfiguration conf = new CapacitySchedulerConfiguration(); + CSPlainConfiguration conf = new CSPlainConfiguration(); setupQueueConfiguration(conf); conf.setClass(YarnConfiguration.RM_SCHEDULER, CapacityScheduler.class, ResourceScheduler.class); @@ -1494,13 +1494,13 @@ protected RMNodeLabelsManager createNodeLabelManager() { return mgr; } }; - CapacitySchedulerConfiguration csConf = - new CapacitySchedulerConfiguration(); + CSPlainConfiguration csConf = + new CSPlainConfiguration(); setupQueueConfiguration(csConf); StringBuilder qState = new StringBuilder(); - qState.append(CapacitySchedulerConfiguration.PREFIX).append(B) - .append(CapacitySchedulerConfiguration.DOT) - .append(CapacitySchedulerConfiguration.STATE); + qState.append(CSPlainConfiguration.PREFIX).append(B) + .append(CSPlainConfiguration.DOT) + .append(CSPlainConfiguration.STATE); csConf.set(qState.toString(), QueueState.STOPPED.name()); YarnConfiguration conf = new YarnConfiguration(csConf); conf.setClass(YarnConfiguration.RM_SCHEDULER, CapacityScheduler.class, @@ -2017,9 +2017,9 @@ public void testKillAllAppsInvalidSource() throws Exception { // that have no CPU available when using the DominantResourceCalculator @Test(timeout = 30000) public void testAppReservationWithDominantResourceCalculator() throws Exception { - CapacitySchedulerConfiguration csconf = - new CapacitySchedulerConfiguration(); - csconf.setResourceComparator(DominantResourceCalculator.class); + CSPlainConfiguration csconf = + new CSPlainConfiguration(); + csconf.setResourceCalculator(DominantResourceCalculator.class); YarnConfiguration conf = new YarnConfiguration(csconf); conf.setClass(YarnConfiguration.RM_SCHEDULER, CapacityScheduler.class, @@ -2079,7 +2079,7 @@ public void testAppReservationWithDominantResourceCalculator() throws Exception @Test public void testPreemptionDisabled() throws Exception { CapacityScheduler cs = new CapacityScheduler(); - CapacitySchedulerConfiguration conf = new CapacitySchedulerConfiguration(); + CSPlainConfiguration conf = new CSPlainConfiguration(); conf.setBoolean(YarnConfiguration.RM_SCHEDULER_ENABLE_MONITORS, true); RMContextImpl rmContext = new RMContextImpl(null, null, null, null, null, null, new RMContainerTokenSecretManager(conf), @@ -2133,7 +2133,7 @@ public void testRefreshQueuesMaxAllocationRefresh() throws Exception { // queue refresh should not allow changing the maximum allocation setting // per queue to be smaller than previous setting CapacityScheduler cs = new CapacityScheduler(); - CapacitySchedulerConfiguration conf = new CapacitySchedulerConfiguration(); + CSPlainConfiguration conf = new CSPlainConfiguration(); setupQueueConfiguration(conf); cs.setConf(new YarnConfiguration()); cs.setRMContext(resourceManager.getRMContext()); @@ -2188,7 +2188,7 @@ public void testRefreshQueuesMaxAllocationPerQueueLarge() throws Exception { CapacityScheduler cs = new CapacityScheduler(); cs.setConf(new YarnConfiguration()); cs.setRMContext(resourceManager.getRMContext()); - CapacitySchedulerConfiguration conf = new CapacitySchedulerConfiguration(); + CSPlainConfiguration conf = new CSPlainConfiguration(); setupQueueConfiguration(conf); cs.init(conf); cs.start(); @@ -2224,7 +2224,7 @@ public void testRefreshQueuesMaxAllocationRefreshLarger() throws Exception { CapacityScheduler cs = new CapacityScheduler(); cs.setConf(new YarnConfiguration()); cs.setRMContext(resourceManager.getRMContext()); - CapacitySchedulerConfiguration conf = new CapacitySchedulerConfiguration(); + CSPlainConfiguration conf = new CSPlainConfiguration(); setupQueueConfiguration(conf); setMaxAllocMb(conf, YarnConfiguration.DEFAULT_RM_SCHEDULER_MAXIMUM_ALLOCATION_MB); @@ -2296,7 +2296,7 @@ public void testRefreshQueuesMaxAllocationCSError() throws Exception { CapacityScheduler cs = new CapacityScheduler(); cs.setConf(new YarnConfiguration()); cs.setRMContext(resourceManager.getRMContext()); - CapacitySchedulerConfiguration conf = new CapacitySchedulerConfiguration(); + CSPlainConfiguration conf = new CSPlainConfiguration(); setupQueueConfiguration(conf); setMaxAllocMb(conf, 10240); setMaxAllocVcores(conf, 10); @@ -2342,7 +2342,7 @@ public void testRefreshQueuesMaxAllocationCSLarger() throws Exception { CapacityScheduler cs = new CapacityScheduler(); cs.setConf(new YarnConfiguration()); cs.setRMContext(resourceManager.getRMContext()); - CapacitySchedulerConfiguration conf = new CapacitySchedulerConfiguration(); + CSPlainConfiguration conf = new CSPlainConfiguration(); setupQueueConfiguration(conf); setMaxAllocMb(conf, 10240); setMaxAllocVcores(conf, 10); @@ -2406,10 +2406,10 @@ private void setMaxAllocMb(Configuration conf, int maxAllocMb) { maxAllocMb); } - private void setMaxAllocMb(CapacitySchedulerConfiguration conf, + private void setMaxAllocMb(CSPlainConfiguration conf, String queueName, int maxAllocMb) { - String propName = CapacitySchedulerConfiguration.getQueuePrefix(queueName) - + CapacitySchedulerConfiguration.MAXIMUM_ALLOCATION_MB; + String propName = CSPlainConfiguration.getQueuePrefix(queueName) + + CSPlainConfiguration.MAXIMUM_ALLOCATION_MB; conf.setInt(propName, maxAllocMb); } @@ -2418,10 +2418,10 @@ private void setMaxAllocVcores(Configuration conf, int maxAllocVcores) { maxAllocVcores); } - private void setMaxAllocVcores(CapacitySchedulerConfiguration conf, + private void setMaxAllocVcores(CSPlainConfiguration conf, String queueName, int maxAllocVcores) { - String propName = CapacitySchedulerConfiguration.getQueuePrefix(queueName) - + CapacitySchedulerConfiguration.MAXIMUM_ALLOCATION_VCORES; + String propName = CSPlainConfiguration.getQueuePrefix(queueName) + + CSPlainConfiguration.MAXIMUM_ALLOCATION_VCORES; conf.setInt(propName, maxAllocVcores); } } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacitySchedulerDynamicBehavior.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacitySchedulerDynamicBehavior.java index ce3382f..62ad80b 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacitySchedulerDynamicBehavior.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacitySchedulerDynamicBehavior.java @@ -42,8 +42,8 @@ public class TestCapacitySchedulerDynamicBehavior { private static final Log LOG = LogFactory .getLog(TestCapacitySchedulerDynamicBehavior.class); - private static final String A = CapacitySchedulerConfiguration.ROOT + ".a"; - private static final String B = CapacitySchedulerConfiguration.ROOT + ".b"; + private static final String A = CSPlainConfiguration.ROOT + ".a"; + private static final String B = CSPlainConfiguration.ROOT + ".b"; private static final String B1 = B + ".b1"; private static final String B2 = B + ".b2"; private static final String B3 = B + ".b3"; @@ -63,7 +63,7 @@ @Before public void setUp() { - CapacitySchedulerConfiguration conf = new CapacitySchedulerConfiguration(); + CSPlainConfiguration conf = new CSPlainConfiguration(); setupPlanQueueConfiguration(conf); conf.setClass(YarnConfiguration.RM_SCHEDULER, CapacityScheduler.class, ResourceScheduler.class); @@ -96,7 +96,7 @@ public void testRefreshQueuesWithReservations() throws Exception { CapacitySchedulerConfiguration conf = cs.getConfiguration(); conf.setCapacity(A, 80f); conf.setCapacity(B, 20f); - cs.reinitialize(conf, rm.getRMContext()); + cs.reinitialize((CSPlainConfiguration)conf, rm.getRMContext()); tcs.checkQueueCapacities(cs, 80f, 20f); } @@ -256,9 +256,9 @@ public void testMoveAppToPlanQueue() throws Exception { rm.stop(); } - private void setupPlanQueueConfiguration(CapacitySchedulerConfiguration conf) { + private void setupPlanQueueConfiguration(CSPlainConfiguration conf) { - conf.setQueues(CapacitySchedulerConfiguration.ROOT, + conf.setQueues(CSPlainConfiguration.ROOT, new String[] { "a", "b" }); conf.setCapacity(A, A_CAPACITY); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacitySchedulerNodeLabelUpdate.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacitySchedulerNodeLabelUpdate.java index 1e339d9..cdaffcb 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacitySchedulerNodeLabelUpdate.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacitySchedulerNodeLabelUpdate.java @@ -59,8 +59,8 @@ public void setUp() throws Exception { } private Configuration getConfigurationWithQueueLabels(Configuration config) { - CapacitySchedulerConfiguration conf = - new CapacitySchedulerConfiguration(config); + CSPlainConfiguration conf = + new CSPlainConfiguration(config); // Define top-level queues conf.setQueues(CapacitySchedulerConfiguration.ROOT, new String[] {"a"}); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacitySchedulerQueueACLs.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacitySchedulerQueueACLs.java index 14ea21b..84d1309 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacitySchedulerQueueACLs.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacitySchedulerQueueACLs.java @@ -29,8 +29,8 @@ public class TestCapacitySchedulerQueueACLs extends QueueACLsTestBase { @Override protected Configuration createConfiguration() { - CapacitySchedulerConfiguration csConf = - new CapacitySchedulerConfiguration(); + CSPlainConfiguration csConf = + new CSPlainConfiguration(); csConf.setQueues(CapacitySchedulerConfiguration.ROOT, new String[] { QUEUEA, QUEUEB }); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestChildQueueOrder.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestChildQueueOrder.java index af58a43..6911801 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestChildQueueOrder.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestChildQueueOrder.java @@ -82,7 +82,7 @@ public void setUp() throws Exception { rmContext = TestUtils.getMockRMContext(); conf = new YarnConfiguration(); - csConf = new CapacitySchedulerConfiguration(); + csConf = new CSPlainConfiguration(); csContext = mock(CapacitySchedulerContext.class); when(csContext.getConf()).thenReturn(conf); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestContainerAllocation.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestContainerAllocation.java index 169517d..9f78b38 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestContainerAllocation.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestContainerAllocation.java @@ -327,17 +327,17 @@ private Configuration getConfigurationWithDefaultQueueLabels( final String A = CapacitySchedulerConfiguration.ROOT + ".a"; final String B = CapacitySchedulerConfiguration.ROOT + ".b"; - CapacitySchedulerConfiguration conf = - (CapacitySchedulerConfiguration) getConfigurationWithQueueLabels(config); - new CapacitySchedulerConfiguration(config); + CSPlainConfiguration conf = + (CSPlainConfiguration) getConfigurationWithQueueLabels(config); + new CSPlainConfiguration(config); conf.setDefaultNodeLabelExpression(A, "x"); conf.setDefaultNodeLabelExpression(B, "y"); return conf; } private Configuration getConfigurationWithQueueLabels(Configuration config) { - CapacitySchedulerConfiguration conf = - new CapacitySchedulerConfiguration(config); + CSPlainConfiguration conf = + new CSPlainConfiguration(config); // Define top-level queues conf.setQueues(CapacitySchedulerConfiguration.ROOT, new String[] {"a", "b", "c"}); @@ -402,8 +402,8 @@ private void checkTaskContainersHost(ApplicationAttemptId attemptId, private Configuration getComplexConfigurationWithQueueLabels( Configuration config) { - CapacitySchedulerConfiguration conf = - new CapacitySchedulerConfiguration(config); + CSPlainConfiguration conf = + new CSPlainConfiguration(config); // Define top-level queues conf.setQueues(CapacitySchedulerConfiguration.ROOT, new String[] {"a", "b"}); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestLeafQueue.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestLeafQueue.java index ead5719..99ac6a8 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestLeafQueue.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestLeafQueue.java @@ -103,7 +103,7 @@ RMContext spyRMContext; ResourceRequest amResourceRequest; CapacityScheduler cs; - CapacitySchedulerConfiguration csConf; + CSPlainConfiguration csConf; CapacitySchedulerContext csContext; CSQueue root; @@ -133,7 +133,7 @@ public void setUp() throws Exception { when(spyRMContext.getRMApps()).thenReturn(spyApps); csConf = - new CapacitySchedulerConfiguration(); + new CSPlainConfiguration(); csConf.setBoolean("yarn.scheduler.capacity.user-metrics.enable", true); final String newRoot = "root" + System.currentTimeMillis(); setupQueueConfiguration(csConf, newRoot); @@ -1949,7 +1949,7 @@ public void testActivateApplicationAfterQueueRefresh() throws Exception { assertEquals(2, e.activeApplications.size()); assertEquals(1, e.pendingApplications.size()); - csConf.setDouble(CapacitySchedulerConfiguration + csConf.setDouble(CSPlainConfiguration .MAXIMUM_APPLICATION_MASTERS_RESOURCE_PERCENT, CapacitySchedulerConfiguration .DEFAULT_MAXIMUM_APPLICATIONMASTERS_RESOURCE_PERCENT * 2); @@ -2300,12 +2300,12 @@ public void testLocalityConstraints() throws Exception { @Test public void testMaxAMResourcePerQueuePercentAfterQueueRefresh() throws Exception { - CapacitySchedulerConfiguration csConf = new CapacitySchedulerConfiguration(); + CSPlainConfiguration csConf = new CSPlainConfiguration(); Resource clusterResource = Resources .createResource(100 * 16 * GB, 100 * 32); CapacitySchedulerContext csContext = mockCSContext(csConf, clusterResource); when(csContext.getRMContext()).thenReturn(rmContext); - csConf.setFloat(CapacitySchedulerConfiguration. + csConf.setFloat(CSPlainConfiguration. MAXIMUM_APPLICATION_MASTERS_RESOURCE_PERCENT, 0.1f); ParentQueue root = new ParentQueue(csContext, CapacitySchedulerConfiguration.ROOT, null, null); @@ -2314,7 +2314,7 @@ public void testMaxAMResourcePerQueuePercentAfterQueueRefresh() assertEquals(0.1f, a.getMaxAMResourcePerQueuePercent(), 1e-3f); assertEquals(a.getAMResourceLimit(), Resources.createResource(160 * GB, 1)); - csConf.setFloat(CapacitySchedulerConfiguration. + csConf.setFloat(CSPlainConfiguration. MAXIMUM_APPLICATION_MASTERS_RESOURCE_PERCENT, 0.2f); LeafQueue newA = new LeafQueue(csContext, A, root, null); a.reinitialize(newA, clusterResource); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestParentQueue.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestParentQueue.java index 696ad7a..04bbebe 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestParentQueue.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestParentQueue.java @@ -78,7 +78,7 @@ public void setUp() throws Exception { rmContext = TestUtils.getMockRMContext(); conf = new YarnConfiguration(); - csConf = new CapacitySchedulerConfiguration(); + csConf = new CSPlainConfiguration(); csContext = mock(CapacitySchedulerContext.class); when(csContext.getConf()).thenReturn(conf); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestQueueMappings.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestQueueMappings.java index 2317fab..2babed8 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestQueueMappings.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestQueueMappings.java @@ -70,8 +70,8 @@ private void setupQueueConfiguration(CapacitySchedulerConfiguration conf) { @Test (timeout = 60000) public void testQueueMapping() throws Exception { - CapacitySchedulerConfiguration csConf = - new CapacitySchedulerConfiguration(); + CSPlainConfiguration csConf = + new CSPlainConfiguration(); setupQueueConfiguration(csConf); YarnConfiguration conf = new YarnConfiguration(csConf); CapacityScheduler cs = new CapacityScheduler(); @@ -134,7 +134,7 @@ public void testQueueMapping() throws Exception { cs.reinitialize(conf, null); checkQMapping("a", Q1, cs); - csConf = new CapacitySchedulerConfiguration(); + csConf = new CSPlainConfiguration(); setupQueueConfiguration(csConf); conf = new YarnConfiguration(csConf); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestQueueParsing.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestQueueParsing.java index 34f7c2c..44d8aaf 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestQueueParsing.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestQueueParsing.java @@ -54,8 +54,8 @@ public void setup() { @Test public void testQueueParsing() throws Exception { - CapacitySchedulerConfiguration csConf = - new CapacitySchedulerConfiguration(); + CSPlainConfiguration csConf = + new CSPlainConfiguration(); setupQueueConfiguration(csConf); YarnConfiguration conf = new YarnConfiguration(csConf); @@ -83,68 +83,68 @@ public void testQueueParsing() throws Exception { } private void setupQueueConfigurationWithSpacesShouldBeTrimmed( - CapacitySchedulerConfiguration conf) { + CSPlainConfiguration conf) { // Define top-level queues conf.set( - CapacitySchedulerConfiguration - .getQueuePrefix(CapacitySchedulerConfiguration.ROOT) - + CapacitySchedulerConfiguration.QUEUES, " a ,b, c"); + CSPlainConfiguration + .getQueuePrefix(CSPlainConfiguration.ROOT) + + CSPlainConfiguration.QUEUES, " a ,b, c"); - final String A = CapacitySchedulerConfiguration.ROOT + ".a"; + final String A = CSPlainConfiguration.ROOT + ".a"; conf.setCapacity(A, 10); conf.setMaximumCapacity(A, 15); - final String B = CapacitySchedulerConfiguration.ROOT + ".b"; + final String B = CSPlainConfiguration.ROOT + ".b"; conf.setCapacity(B, 20); - final String C = CapacitySchedulerConfiguration.ROOT + ".c"; + final String C = CSPlainConfiguration.ROOT + ".c"; conf.setCapacity(C, 70); conf.setMaximumCapacity(C, 70); } private void setupNestedQueueConfigurationWithSpacesShouldBeTrimmed( - CapacitySchedulerConfiguration conf) { + CSPlainConfiguration conf) { // Define top-level queues conf.set( - CapacitySchedulerConfiguration - .getQueuePrefix(CapacitySchedulerConfiguration.ROOT) - + CapacitySchedulerConfiguration.QUEUES, " a ,b, c"); + CSPlainConfiguration + .getQueuePrefix(CSPlainConfiguration.ROOT) + + CSPlainConfiguration.QUEUES, " a ,b, c"); - final String A = CapacitySchedulerConfiguration.ROOT + ".a"; + final String A = CSPlainConfiguration.ROOT + ".a"; conf.setCapacity(A, 10); conf.setMaximumCapacity(A, 15); - final String B = CapacitySchedulerConfiguration.ROOT + ".b"; + final String B = CSPlainConfiguration.ROOT + ".b"; conf.setCapacity(B, 20); - final String C = CapacitySchedulerConfiguration.ROOT + ".c"; + final String C = CSPlainConfiguration.ROOT + ".c"; conf.setCapacity(C, 70); conf.setMaximumCapacity(C, 70); // sub queues for A - conf.set(CapacitySchedulerConfiguration.getQueuePrefix(A) - + CapacitySchedulerConfiguration.QUEUES, "a1, a2 "); + conf.set(CSPlainConfiguration.getQueuePrefix(A) + + CSPlainConfiguration.QUEUES, "a1, a2 "); - final String A1 = CapacitySchedulerConfiguration.ROOT + ".a.a1"; + final String A1 = CSPlainConfiguration.ROOT + ".a.a1"; conf.setCapacity(A1, 60); - final String A2 = CapacitySchedulerConfiguration.ROOT + ".a.a2"; + final String A2 = CSPlainConfiguration.ROOT + ".a.a2"; conf.setCapacity(A2, 40); } - private void setupQueueConfiguration(CapacitySchedulerConfiguration conf) { + private void setupQueueConfiguration(CSPlainConfiguration conf) { // Define top-level queues - conf.setQueues(CapacitySchedulerConfiguration.ROOT, new String[] {"a", "b", "c"}); + conf.setQueues(CSPlainConfiguration.ROOT, new String[] {"a", "b", "c"}); - final String A = CapacitySchedulerConfiguration.ROOT + ".a"; + final String A = CSPlainConfiguration.ROOT + ".a"; conf.setCapacity(A, 10); conf.setMaximumCapacity(A, 15); - final String B = CapacitySchedulerConfiguration.ROOT + ".b"; + final String B = CSPlainConfiguration.ROOT + ".b"; conf.setCapacity(B, 20); - final String C = CapacitySchedulerConfiguration.ROOT + ".c"; + final String C = CSPlainConfiguration.ROOT + ".c"; conf.setCapacity(C, 70); conf.setMaximumCapacity(C, 70); @@ -203,10 +203,10 @@ private void setupQueueConfiguration(CapacitySchedulerConfiguration conf) { @Test (expected=java.lang.IllegalArgumentException.class) public void testRootQueueParsing() throws Exception { - CapacitySchedulerConfiguration conf = new CapacitySchedulerConfiguration(); + CSPlainConfiguration conf = new CSPlainConfiguration(); // non-100 percent value will throw IllegalArgumentException - conf.setCapacity(CapacitySchedulerConfiguration.ROOT, 90); + conf.setCapacity(CSPlainConfiguration.ROOT, 90); CapacityScheduler capacityScheduler = new CapacityScheduler(); capacityScheduler.setConf(new YarnConfiguration()); @@ -217,15 +217,15 @@ public void testRootQueueParsing() throws Exception { } public void testMaxCapacity() throws Exception { - CapacitySchedulerConfiguration conf = new CapacitySchedulerConfiguration(); + CSPlainConfiguration conf = new CSPlainConfiguration(); - conf.setQueues(CapacitySchedulerConfiguration.ROOT, new String[] {"a", "b", "c"}); + conf.setQueues(CSPlainConfiguration.ROOT, new String[] {"a", "b", "c"}); - final String A = CapacitySchedulerConfiguration.ROOT + ".a"; + final String A = CSPlainConfiguration.ROOT + ".a"; conf.setCapacity(A, 50); conf.setMaximumCapacity(A, 60); - final String B = CapacitySchedulerConfiguration.ROOT + ".b"; + final String B = CSPlainConfiguration.ROOT + ".b"; conf.setCapacity(B, 50); conf.setMaximumCapacity(B, 45); // Should throw an exception @@ -265,15 +265,15 @@ public void testMaxCapacity() throws Exception { capacityScheduler.stop(); } - private void setupQueueConfigurationWithoutLabels(CapacitySchedulerConfiguration conf) { + private void setupQueueConfigurationWithoutLabels(CSPlainConfiguration conf) { // Define top-level queues - conf.setQueues(CapacitySchedulerConfiguration.ROOT, new String[] {"a", "b"}); + conf.setQueues(CSPlainConfiguration.ROOT, new String[] {"a", "b"}); - final String A = CapacitySchedulerConfiguration.ROOT + ".a"; + final String A = CSPlainConfiguration.ROOT + ".a"; conf.setCapacity(A, 10); conf.setMaximumCapacity(A, 15); - final String B = CapacitySchedulerConfiguration.ROOT + ".b"; + final String B = CSPlainConfiguration.ROOT + ".b"; conf.setCapacity(B, 90); LOG.info("Setup top-level queues"); @@ -299,17 +299,17 @@ private void setupQueueConfigurationWithoutLabels(CapacitySchedulerConfiguration conf.setMaximumCapacity(B3, 35); } - private void setupQueueConfigurationWithLabels(CapacitySchedulerConfiguration conf) { + private void setupQueueConfigurationWithLabels(CSPlainConfiguration conf) { // Define top-level queues - conf.setQueues(CapacitySchedulerConfiguration.ROOT, new String[] {"a", "b"}); - conf.setCapacityByLabel(CapacitySchedulerConfiguration.ROOT, "red", 100); - conf.setCapacityByLabel(CapacitySchedulerConfiguration.ROOT, "blue", 100); + conf.setQueues(CSPlainConfiguration.ROOT, new String[] {"a", "b"}); + conf.setCapacityByLabel(CSPlainConfiguration.ROOT, "red", 100); + conf.setCapacityByLabel(CSPlainConfiguration.ROOT, "blue", 100); - final String A = CapacitySchedulerConfiguration.ROOT + ".a"; + final String A = CSPlainConfiguration.ROOT + ".a"; conf.setCapacity(A, 10); conf.setMaximumCapacity(A, 15); - final String B = CapacitySchedulerConfiguration.ROOT + ".b"; + final String B = CSPlainConfiguration.ROOT + ".b"; conf.setCapacity(B, 90); LOG.info("Setup top-level queues"); @@ -359,14 +359,14 @@ private void setupQueueConfigurationWithLabels(CapacitySchedulerConfiguration co } private void setupQueueConfigurationWithLabelsInherit( - CapacitySchedulerConfiguration conf) { + CSPlainConfiguration conf) { // Define top-level queues - conf.setQueues(CapacitySchedulerConfiguration.ROOT, new String[] {"a", "b"}); - conf.setCapacityByLabel(CapacitySchedulerConfiguration.ROOT, "red", 100); - conf.setCapacityByLabel(CapacitySchedulerConfiguration.ROOT, "blue", 100); + conf.setQueues(CSPlainConfiguration.ROOT, new String[] {"a", "b"}); + conf.setCapacityByLabel(CSPlainConfiguration.ROOT, "red", 100); + conf.setCapacityByLabel(CSPlainConfiguration.ROOT, "blue", 100); // Set A configuration - final String A = CapacitySchedulerConfiguration.ROOT + ".a"; + final String A = CSPlainConfiguration.ROOT + ".a"; conf.setCapacity(A, 10); conf.setMaximumCapacity(A, 15); conf.setQueues(A, new String[] {"a1", "a2"}); @@ -375,7 +375,7 @@ private void setupQueueConfigurationWithLabelsInherit( conf.setCapacityByLabel(A, "blue", 100); // Set B configuraiton - final String B = CapacitySchedulerConfiguration.ROOT + ".b"; + final String B = CSPlainConfiguration.ROOT + ".b"; conf.setCapacity(B, 90); conf.setAccessibleNodeLabels(B, CommonNodeLabelsManager.EMPTY_STRING_SET); @@ -395,12 +395,12 @@ private void setupQueueConfigurationWithLabelsInherit( } private void setupQueueConfigurationWithSingleLevel( - CapacitySchedulerConfiguration conf) { + CSPlainConfiguration conf) { // Define top-level queues - conf.setQueues(CapacitySchedulerConfiguration.ROOT, new String[] {"a", "b"}); + conf.setQueues(CSPlainConfiguration.ROOT, new String[] {"a", "b"}); // Set A configuration - final String A = CapacitySchedulerConfiguration.ROOT + ".a"; + final String A = CSPlainConfiguration.ROOT + ".a"; conf.setCapacity(A, 10); conf.setMaximumCapacity(A, 15); conf.setAccessibleNodeLabels(A, ImmutableSet.of("red", "blue")); @@ -408,7 +408,7 @@ private void setupQueueConfigurationWithSingleLevel( conf.setCapacityByLabel(A, "blue", 90); // Set B configuraiton - final String B = CapacitySchedulerConfiguration.ROOT + ".b"; + final String B = CSPlainConfiguration.ROOT + ".b"; conf.setCapacity(B, 90); conf.setAccessibleNodeLabels(B, ImmutableSet.of("red", "blue")); conf.setCapacityByLabel(B, "red", 10); @@ -418,8 +418,8 @@ private void setupQueueConfigurationWithSingleLevel( @Test public void testQueueParsingReinitializeWithLabels() throws IOException { nodeLabelManager.addToCluserNodeLabels(ImmutableSet.of("red", "blue")); - CapacitySchedulerConfiguration csConf = - new CapacitySchedulerConfiguration(); + CSPlainConfiguration csConf = + new CSPlainConfiguration(); setupQueueConfigurationWithoutLabels(csConf); YarnConfiguration conf = new YarnConfiguration(csConf); @@ -434,7 +434,7 @@ public void testQueueParsingReinitializeWithLabels() throws IOException { capacityScheduler.setRMContext(rmContext); capacityScheduler.init(conf); capacityScheduler.start(); - csConf = new CapacitySchedulerConfiguration(); + csConf = new CSPlainConfiguration(); setupQueueConfigurationWithLabels(csConf); conf = new YarnConfiguration(csConf); capacityScheduler.reinitialize(conf, rmContext); @@ -508,8 +508,8 @@ public void testQueueParsingWithLabels() throws IOException { nodeLabelManager.addToCluserNodeLabels(ImmutableSet.of("red", "blue")); YarnConfiguration conf = new YarnConfiguration(); - CapacitySchedulerConfiguration csConf = - new CapacitySchedulerConfiguration(conf); + CSPlainConfiguration csConf = + new CSPlainConfiguration(conf); setupQueueConfigurationWithLabels(csConf); CapacityScheduler capacityScheduler = new CapacityScheduler(); @@ -532,8 +532,8 @@ public void testQueueParsingWithLabelsInherit() throws IOException { nodeLabelManager.addToCluserNodeLabels(ImmutableSet.of("red", "blue")); YarnConfiguration conf = new YarnConfiguration(); - CapacitySchedulerConfiguration csConf = - new CapacitySchedulerConfiguration(conf); + CSPlainConfiguration csConf = + new CSPlainConfiguration(conf); setupQueueConfigurationWithLabelsInherit(csConf); CapacityScheduler capacityScheduler = new CapacityScheduler(); @@ -555,8 +555,8 @@ public void testQueueParsingWithLabelsInherit() throws IOException { public void testQueueParsingWhenLabelsNotExistedInNodeLabelManager() throws IOException { YarnConfiguration conf = new YarnConfiguration(); - CapacitySchedulerConfiguration csConf = - new CapacitySchedulerConfiguration(conf); + CSPlainConfiguration csConf = + new CSPlainConfiguration(conf); setupQueueConfigurationWithLabels(csConf); CapacityScheduler capacityScheduler = new CapacityScheduler(); @@ -583,8 +583,8 @@ public void testQueueParsingWhenLabelsNotExistedInNodeLabelManager() public void testQueueParsingWhenLabelsInheritedNotExistedInNodeLabelManager() throws IOException { YarnConfiguration conf = new YarnConfiguration(); - CapacitySchedulerConfiguration csConf = - new CapacitySchedulerConfiguration(conf); + CSPlainConfiguration csConf = + new CSPlainConfiguration(conf); setupQueueConfigurationWithLabelsInherit(csConf); CapacityScheduler capacityScheduler = new CapacityScheduler(); @@ -611,8 +611,8 @@ public void testQueueParsingWhenLabelsInheritedNotExistedInNodeLabelManager() public void testSingleLevelQueueParsingWhenLabelsNotExistedInNodeLabelManager() throws IOException { YarnConfiguration conf = new YarnConfiguration(); - CapacitySchedulerConfiguration csConf = - new CapacitySchedulerConfiguration(conf); + CSPlainConfiguration csConf = + new CSPlainConfiguration(conf); setupQueueConfigurationWithSingleLevel(csConf); CapacityScheduler capacityScheduler = new CapacityScheduler(); @@ -638,8 +638,8 @@ public void testSingleLevelQueueParsingWhenLabelsNotExistedInNodeLabelManager() @Test(expected = Exception.class) public void testQueueParsingWhenLabelsNotExist() throws IOException { YarnConfiguration conf = new YarnConfiguration(); - CapacitySchedulerConfiguration csConf = - new CapacitySchedulerConfiguration(conf); + CSPlainConfiguration csConf = + new CSPlainConfiguration(conf); setupQueueConfigurationWithLabels(csConf); CapacityScheduler capacityScheduler = new CapacityScheduler(); @@ -670,10 +670,10 @@ public void testQueueParsingWithUnusedLabels() throws IOException { // shouldn't fail nodeLabelManager.addToCluserNodeLabels(labels); - CapacitySchedulerConfiguration csConf = - new CapacitySchedulerConfiguration(); + CSPlainConfiguration csConf = + new CSPlainConfiguration(); setupQueueConfiguration(csConf); - csConf.setAccessibleNodeLabels(CapacitySchedulerConfiguration.ROOT, labels); + csConf.setAccessibleNodeLabels(CSPlainConfiguration.ROOT, labels); YarnConfiguration conf = new YarnConfiguration(csConf); CapacityScheduler capacityScheduler = new CapacityScheduler(); @@ -690,7 +690,7 @@ public void testQueueParsingWithUnusedLabels() throws IOException { capacityScheduler.reinitialize(conf, rmContext); // check root queue's capacity by label -- they should be all zero - CSQueue root = capacityScheduler.getQueue(CapacitySchedulerConfiguration.ROOT); + CSQueue root = capacityScheduler.getQueue(CSPlainConfiguration.ROOT); Assert.assertEquals(0, root.getQueueCapacities().getCapacity("red"), DELTA); Assert.assertEquals(0, root.getQueueCapacities().getCapacity("blue"), DELTA); @@ -712,8 +712,8 @@ public void testQueueParsingWithUnusedLabels() throws IOException { @Test public void testQueueParsingShouldTrimSpaces() throws Exception { - CapacitySchedulerConfiguration csConf = - new CapacitySchedulerConfiguration(); + CSPlainConfiguration csConf = + new CSPlainConfiguration(); setupQueueConfigurationWithSpacesShouldBeTrimmed(csConf); YarnConfiguration conf = new YarnConfiguration(csConf); @@ -737,8 +737,8 @@ public void testQueueParsingShouldTrimSpaces() throws Exception { @Test public void testNestedQueueParsingShouldTrimSpaces() throws Exception { - CapacitySchedulerConfiguration csConf = - new CapacitySchedulerConfiguration(); + CSPlainConfiguration csConf = + new CSPlainConfiguration(); setupNestedQueueConfigurationWithSpacesShouldBeTrimmed(csConf); YarnConfiguration conf = new YarnConfiguration(csConf); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestReservationQueue.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestReservationQueue.java index 4e6c73d..0d582f3 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestReservationQueue.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestReservationQueue.java @@ -47,7 +47,7 @@ @Before public void setup() throws IOException { // setup a context / conf - csConf = new CapacitySchedulerConfiguration(); + csConf = new CSPlainConfiguration(); YarnConfiguration conf = new YarnConfiguration(); csContext = mock(CapacitySchedulerContext.class); when(csContext.getConfiguration()).thenReturn(csConf); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestReservations.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestReservations.java index 985609e..d6bd245 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestReservations.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestReservations.java @@ -79,7 +79,7 @@ RMContext rmContext; RMContext spyRMContext; CapacityScheduler cs; - // CapacitySchedulerConfiguration csConf; + // CSPlainConfiguration csConf; CapacitySchedulerContext csContext; private final ResourceCalculator resourceCalculator = new DefaultResourceCalculator(); @@ -99,7 +99,7 @@ public void setUp() throws Exception { } - private void setup(CapacitySchedulerConfiguration csConf) throws Exception { + private void setup(CSPlainConfiguration csConf) throws Exception { csConf.setBoolean("yarn.scheduler.capacity.user-metrics.enable", true); final String newRoot = "root" + System.currentTimeMillis(); @@ -131,7 +131,7 @@ private void setup(CapacitySchedulerConfiguration csConf) throws Exception { containerTokenSecretManager); root = CapacityScheduler.parseQueue(csContext, csConf, null, - CapacitySchedulerConfiguration.ROOT, queues, queues, TestUtils.spyHook); + CSPlainConfiguration.ROOT, queues, queues, TestUtils.spyHook); spyRMContext = spy(rmContext); when(spyRMContext.getScheduler()).thenReturn(cs); @@ -143,17 +143,17 @@ private void setup(CapacitySchedulerConfiguration csConf) throws Exception { private static final String A = "a"; - private void setupQueueConfiguration(CapacitySchedulerConfiguration conf, + private void setupQueueConfiguration(CSPlainConfiguration conf, final String newRoot) { // Define top-level queues - conf.setQueues(CapacitySchedulerConfiguration.ROOT, + conf.setQueues(CSPlainConfiguration.ROOT, new String[] { newRoot }); - conf.setMaximumCapacity(CapacitySchedulerConfiguration.ROOT, 100); - conf.setAcl(CapacitySchedulerConfiguration.ROOT, + conf.setMaximumCapacity(CSPlainConfiguration.ROOT, 100); + conf.setAcl(CSPlainConfiguration.ROOT, QueueACL.SUBMIT_APPLICATIONS, " "); - final String Q_newRoot = CapacitySchedulerConfiguration.ROOT + "." + final String Q_newRoot = CSPlainConfiguration.ROOT + "." + newRoot; conf.setQueues(Q_newRoot, new String[] { A }); conf.setCapacity(Q_newRoot, 100); @@ -203,7 +203,7 @@ public Container answer(InvocationOnMock invocation) throws Throwable { public void testReservation() throws Exception { // Test that we now unreserve and use a node that has space - CapacitySchedulerConfiguration csConf = new CapacitySchedulerConfiguration(); + CSPlainConfiguration csConf = new CSPlainConfiguration(); setup(csConf); // Manipulate queue 'a' @@ -350,7 +350,7 @@ public void testReservationNoContinueLook() throws Exception { queues = new HashMap(); // test that the deadlock occurs when turned off - CapacitySchedulerConfiguration csConf = new CapacitySchedulerConfiguration(); + CSPlainConfiguration csConf = new CSPlainConfiguration(); csConf.setBoolean( "yarn.scheduler.capacity.reservations-continue-look-all-nodes", false); setup(csConf); @@ -497,7 +497,7 @@ public void testReservationNoContinueLook() throws Exception { public void testAssignContainersNeedToUnreserve() throws Exception { // Test that we now unreserve and use a node that has space - CapacitySchedulerConfiguration csConf = new CapacitySchedulerConfiguration(); + CSPlainConfiguration csConf = new CSPlainConfiguration(); setup(csConf); // Manipulate queue 'a' @@ -615,7 +615,7 @@ public void testAssignContainersNeedToUnreserve() throws Exception { @Test public void testGetAppToUnreserve() throws Exception { - CapacitySchedulerConfiguration csConf = new CapacitySchedulerConfiguration(); + CSPlainConfiguration csConf = new CSPlainConfiguration(); setup(csConf); final String user_0 = "user_0"; final ApplicationAttemptId appAttemptId_0 = TestUtils @@ -682,7 +682,7 @@ public void testGetAppToUnreserve() throws Exception { @Test public void testFindNodeToUnreserve() throws Exception { - CapacitySchedulerConfiguration csConf = new CapacitySchedulerConfiguration(); + CSPlainConfiguration csConf = new CSPlainConfiguration(); setup(csConf); final String user_0 = "user_0"; final ApplicationAttemptId appAttemptId_0 = TestUtils @@ -733,7 +733,7 @@ public void testFindNodeToUnreserve() throws Exception { @Test public void testAssignToQueue() throws Exception { - CapacitySchedulerConfiguration csConf = new CapacitySchedulerConfiguration(); + CSPlainConfiguration csConf = new CSPlainConfiguration(); setup(csConf); // Manipulate queue 'a' @@ -871,17 +871,17 @@ public void testAssignToQueue() throws Exception { } public void refreshQueuesTurnOffReservationsContLook(LeafQueue a, - CapacitySchedulerConfiguration csConf) throws Exception { + CSPlainConfiguration csConf) throws Exception { // before reinitialization assertEquals(true, a.getReservationContinueLooking()); assertEquals(true, ((ParentQueue) a.getParent()).getReservationContinueLooking()); csConf.setBoolean( - CapacitySchedulerConfiguration.RESERVE_CONT_LOOK_ALL_NODES, false); + CSPlainConfiguration.RESERVE_CONT_LOOK_ALL_NODES, false); Map newQueues = new HashMap(); CSQueue newRoot = CapacityScheduler.parseQueue(csContext, csConf, null, - CapacitySchedulerConfiguration.ROOT, newQueues, queues, + CSPlainConfiguration.ROOT, newQueues, queues, TestUtils.spyHook); queues = newQueues; root.reinitialize(newRoot, cs.getClusterResource()); @@ -895,7 +895,7 @@ public void refreshQueuesTurnOffReservationsContLook(LeafQueue a, @Test public void testContinueLookingReservationsAfterQueueRefresh() throws Exception { - CapacitySchedulerConfiguration csConf = new CapacitySchedulerConfiguration(); + CSPlainConfiguration csConf = new CSPlainConfiguration(); setup(csConf); // Manipulate queue 'e' @@ -907,7 +907,7 @@ public void testContinueLookingReservationsAfterQueueRefresh() @Test public void testAssignToUser() throws Exception { - CapacitySchedulerConfiguration csConf = new CapacitySchedulerConfiguration(); + CSPlainConfiguration csConf = new CSPlainConfiguration(); setup(csConf); // Manipulate queue 'a' @@ -1033,7 +1033,7 @@ public void testAssignToUser() throws Exception { public void testReservationsNoneAvailable() throws Exception { // Test that we now unreserve and use a node that has space - CapacitySchedulerConfiguration csConf = new CapacitySchedulerConfiguration(); + CSPlainConfiguration csConf = new CSPlainConfiguration(); setup(csConf); // Manipulate queue 'a' diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/config/TestBaseSchedulerHierarchyConfiguration.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/config/TestBaseSchedulerHierarchyConfiguration.java new file mode 100644 index 0000000..3f03693 --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/config/TestBaseSchedulerHierarchyConfiguration.java @@ -0,0 +1,85 @@ +package org.apache.hadoop.yarn.server.resourcemanager.scheduler.config; + +import org.apache.hadoop.yarn.exceptions.YarnException; +import org.apache.hadoop.yarn.exceptions.YarnRuntimeException; +import org.junit.Assert; +import org.junit.Test; + +public class TestBaseSchedulerHierarchyConfiguration { + @Test + public void testBaseSchedulerHierarchyConfiguration() throws YarnException { + String content = "" + + "" + + " " + + " " + + " " + + " a-2" + + + " " + + " " + + " a1-1" + + " " + + + " " + + " " + + " " + + " " + + + " " + + " " + + " " + + + " " + + " global-1" + + " global-3" + + " " + + ""; + + SchedulerConfNode root = SchedulerConfParser.parse(content); + BaseSchedulerHierarchyConfiguration conf = + new BaseSchedulerHierarchyConfiguration(root); + + Assert.assertEquals("global-1", conf.getGlobalConfig("param1", null)); + Assert.assertEquals(null, conf.getGlobalConfig("param2", null)); + + // root.a doesn't have param1 specify + Assert.assertEquals(null, conf.getQueueConfig("root.a", "param1", null)); + + // root.a param1 will inherit global-1 + Assert.assertEquals("global-1", + conf.getQueueConfig("root.a", "param1", null, true)); + + // root.a.a1 has param1 + Assert.assertEquals("a1-1", + conf.getQueueConfig("root.a.a1", "param1", null)); + + // root.a.a1 param2 will inherit root.a + Assert.assertEquals("a-2", + conf.getQueueConfig("root.a.a1", "param2", null, true)); + + // root.b will inherit global as well + Assert.assertEquals("global-3", + conf.getQueueConfig("root.b", "param3", null, true)); + + try { + conf.getQueueConfig("root.a.a3", "param1", null); + Assert.fail("try to fetch an non-existed queue, should fail"); + } catch (YarnRuntimeException e) { + // expected + } + + try { + conf.getQueueConfig("root.c", "param1", null); + Assert.fail("try to fetch an non-existed queue, should fail"); + } catch (YarnRuntimeException e) { + // expected + } + + try { + conf.getQueueConfig("non-existed", "param1", null); + Assert.fail("try to fetch an non-existed queue, should fail"); + } catch (YarnRuntimeException e) { + // expected + } + } +} diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/config/TestSchedulerConfNode.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/config/TestSchedulerConfNode.java new file mode 100644 index 0000000..3eccc20 --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/config/TestSchedulerConfNode.java @@ -0,0 +1,4 @@ +package org.apache.hadoop.yarn.server.resourcemanager.scheduler.config; + +public class TestSchedulerConfNode { +} diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/config/TestSchedulerConfParser.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/config/TestSchedulerConfParser.java new file mode 100644 index 0000000..efdff01 --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/config/TestSchedulerConfParser.java @@ -0,0 +1,116 @@ +/** +* Licensed to the Apache Software Foundation (ASF) under one +* or more contributor license agreements. See the NOTICE file +* distributed with this work for additional information +* regarding copyright ownership. The ASF licenses this file +* to you under the Apache License, Version 2.0 (the +* "License"); you may not use this file except in compliance +* with the License. You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, software +* distributed under the License is distributed on an "AS IS" BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions and +* limitations under the License. +*/ + +package org.apache.hadoop.yarn.server.resourcemanager.scheduler.config; + +import java.util.Iterator; + +import org.apache.hadoop.yarn.exceptions.YarnException; +import org.junit.Assert; +import org.junit.Test; + +import com.google.common.collect.ArrayListMultimap; +import com.google.common.collect.ListMultimap; + +public class TestSchedulerConfParser { + @Test + public void testSimpleSchedulerConfParser() throws YarnException { + String content = "" + + "" + + " " + + " " + + " " + + " " + + " value3" + + " value4" + + " " + + " " + + " value5" + + " value6" + + " " + + " " + + " " + + " " + + " value1" + + " value2" + + ""; + + SchedulerConfNode root = SchedulerConfParser.parse(content); + SchedulerConfNode expectedRoot = getExpectedSimpleSchedulerConfNode(); + + assertEquals(expectedRoot, root); + } + + public void assertEquals(SchedulerConfNode left, SchedulerConfNode right) { + Assert.assertEquals(left.getTag(), right.getTag()); + Assert.assertEquals(left.getName(), right.getName()); + Assert.assertEquals(left.getValue(), right.getValue()); + Assert.assertEquals(left.getType(), right.getType()); + Assert.assertEquals(left.getFields().values().size(), right.getFields().values().size()); + + Iterator leftIterator = + left.getFields().values().iterator(); + Iterator rightIterator = + right.getFields().values().iterator(); + while (leftIterator.hasNext()) { + assertEquals(leftIterator.next(), rightIterator.next()); + } + } + + private SchedulerConfNode getExpectedSimpleSchedulerConfNode() { + ListMultimap fields = ArrayListMultimap.create(); + + // leaf1 + fields.put("param3", new SchedulerConfNode("", "", "value3", "param3", null)); + fields.put("param4", new SchedulerConfNode("", "", "value4", "param4", null)); + SchedulerConfNode leaf1 = new SchedulerConfNode("leaf1", "", "", "leaf", fields); + + // leaf2 + fields = ArrayListMultimap.create(); + fields.put("param5", new SchedulerConfNode("", "", "value5", "param5", null)); + fields.put("param6", new SchedulerConfNode("", "", "value6", "param6", null)); + SchedulerConfNode leaf2 = new SchedulerConfNode("", "", "", "leaf", fields); + + // inner children + fields = ArrayListMultimap.create(); + fields.put("leaf", leaf1); + fields.put("leaf", leaf2); + SchedulerConfNode innerChildren = + new SchedulerConfNode("", "", "", "children", fields); + + // parent + fields = ArrayListMultimap.create(); + fields.put("children", innerChildren); + SchedulerConfNode parent = + new SchedulerConfNode("parent", "type", "", "parent", fields); + + // outer children + fields = ArrayListMultimap.create(); + fields.put("parent", parent); + SchedulerConfNode outerChildren = new SchedulerConfNode("", "", "", "children", fields); + + // root + fields = ArrayListMultimap.create(); + fields.put("children", outerChildren); + fields.put("param1", new SchedulerConfNode("", "", "value1", "param1", null)); + fields.put("param2", new SchedulerConfNode("", "", "value2", "param2", null)); + SchedulerConfNode root = new SchedulerConfNode("", "", "", "root", fields); + + return root; + } +} diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebApp.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebApp.java index fb1e61d..1ef94af 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebApp.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebApp.java @@ -43,6 +43,7 @@ import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMApp; import org.apache.hadoop.yarn.server.resourcemanager.rmnode.RMNode; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.ResourceScheduler; +import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CSPlainConfiguration; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacityScheduler; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacitySchedulerConfiguration; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fifo.FifoScheduler; @@ -202,7 +203,7 @@ public static ResourceManager mockRm(RMContext rmContext) throws IOException { public static CapacityScheduler mockCapacityScheduler() throws IOException { // stolen from TestCapacityScheduler - CapacitySchedulerConfiguration conf = new CapacitySchedulerConfiguration(); + CSPlainConfiguration conf = new CSPlainConfiguration(); setupQueueConfiguration(conf); CapacityScheduler cs = new CapacityScheduler(); @@ -284,7 +285,7 @@ public static ResourceManager mockFifoRm(int apps, int racks, int nodes, public static FifoScheduler mockFifoScheduler(RMContext rmContext) throws Exception { - CapacitySchedulerConfiguration conf = new CapacitySchedulerConfiguration(); + CSPlainConfiguration conf = new CSPlainConfiguration(); setupFifoQueueConfiguration(conf); FifoScheduler fs = new FifoScheduler(); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesAppsModification.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesAppsModification.java index 8e5e601..7c04cda 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesAppsModification.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesAppsModification.java @@ -44,6 +44,7 @@ import javax.xml.parsers.ParserConfigurationException; import com.sun.jersey.api.client.config.DefaultClientConfig; + import org.apache.commons.codec.binary.Base64; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.io.Text; @@ -63,14 +64,11 @@ import org.apache.hadoop.yarn.server.resourcemanager.ResourceManager; import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMApp; import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMAppState; +import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CSPlainConfiguration; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacityScheduler; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacitySchedulerConfiguration; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.FairScheduler; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.FairSchedulerConfiguration; -import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.AppState; -import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.ApplicationSubmissionContextInfo; -import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.CredentialsInfo; -import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.LocalResourceInfo; import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.*; import org.apache.hadoop.yarn.util.ConverterUtils; import org.apache.hadoop.yarn.webapp.GenericExceptionHandler; @@ -529,8 +527,8 @@ public void testSingleAppKillUnauthorized() throws Exception { // FairScheduler use ALLOCATION_FILE to configure ACL if (isCapacityScheduler) { // default root queue allows anyone to have admin acl - CapacitySchedulerConfiguration csconf = - new CapacitySchedulerConfiguration(); + CSPlainConfiguration csconf = + new CSPlainConfiguration(); csconf.setAcl("root", QueueACL.ADMINISTER_QUEUE, "someuser"); csconf.setAcl("root.default", QueueACL.ADMINISTER_QUEUE, "someuser"); rm.getResourceScheduler().reinitialize(csconf, rm.getRMContext()); @@ -978,8 +976,8 @@ public void testAppMove() throws Exception { rm.getResourceScheduler() instanceof CapacityScheduler; // default root queue allows anyone to have admin acl - CapacitySchedulerConfiguration csconf = - new CapacitySchedulerConfiguration(); + CSPlainConfiguration csconf = + new CSPlainConfiguration(); String[] queues = { "default", "test" }; csconf.setQueues("root", queues); csconf.setCapacity("root.default", 50.0f); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesCapacitySched.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesCapacitySched.java index ba5c73b..eb86a5a 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesCapacitySched.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesCapacitySched.java @@ -32,6 +32,7 @@ import org.apache.hadoop.yarn.server.resourcemanager.MockRM; import org.apache.hadoop.yarn.server.resourcemanager.ResourceManager; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.ResourceScheduler; +import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CSPlainConfiguration; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacityScheduler; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacitySchedulerConfiguration; import org.apache.hadoop.yarn.util.resource.Resources; @@ -61,7 +62,7 @@ public class TestRMWebServicesCapacitySched extends JerseyTestBase { private static MockRM rm; - private CapacitySchedulerConfiguration csConf; + private CSPlainConfiguration csConf; private YarnConfiguration conf; private class QueueInfo { @@ -92,7 +93,7 @@ protected void configureServlets() { bind(JAXBContextResolver.class); bind(RMWebServices.class); bind(GenericExceptionHandler.class); - csConf = new CapacitySchedulerConfiguration(); + csConf = new CSPlainConfiguration(); setupQueueConfiguration(csConf); conf = new YarnConfiguration(csConf); conf.setClass(YarnConfiguration.RM_SCHEDULER, CapacityScheduler.class,