diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ResourceManager.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ResourceManager.java index d939ac1525e..563c3bc7b66 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ResourceManager.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ResourceManager.java @@ -21,6 +21,10 @@ import com.google.common.annotations.VisibleForTesting; import com.sun.jersey.spi.container.servlet.ServletContainer; +import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.converter.FSConfigToCSConfigArgumentHandler; +import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.converter.FSConfigToCSConfigArgumentHandler.CliOption; +import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.converter.FSConfigToCSConfigConverter; +import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.converter.FSConfigToCSConfigRuleHandler; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.slf4j.Marker; @@ -136,7 +140,6 @@ import java.net.InetSocketAddress; import java.net.URI; import java.net.URL; -import java.net.URLClassLoader; import java.nio.charset.Charset; import java.security.PrivilegedExceptionAction; import java.security.SecureRandom; @@ -226,6 +229,13 @@ private Configuration conf; private UserGroupInformation rmLoginUGI; + private static FSConfigToCSConfigArgumentHandler + fsConfigConversionArgumentHandler; + + static { + FSConfigToCSConfigConverter converter = initFSConfigConverter(); + initFSArgumentHandler(converter); + } public ResourceManager() { super("ResourceManager"); @@ -1556,6 +1566,15 @@ public static void main(String argv[]) { } else if (argv[0].equals("-remove-application-from-state-store") && argv.length == 2) { removeApplication(conf, argv[1]); + } else if (argv[0].equals("-convert-fs-configuration")) { + String[] args = Arrays.copyOfRange(argv, 1, argv.length); + try { + fsConfigConversionArgumentHandler.parseAndConvert(args); + } catch (Throwable t) { + LOG.error(FATAL, + "Error while starting FS configuration conversion!", t); + System.exit(-1); + } } else { printUsage(System.err); } @@ -1666,6 +1685,12 @@ private static void printUsage(PrintStream out) { out.println("Usage: yarn resourcemanager [-format-state-store]"); out.println(" " + "[-remove-application-from-state-store ]" + "\n"); + + out.println("[-convert-fs-configuration "); + for (CliOption cliOption : CliOption.values()) { + out.println(" " + cliOption.getAsArgumentString()); + } + out.println("]"); } protected RMAppLifetimeMonitor createRMAppLifetimeMonitor() { @@ -1683,4 +1708,17 @@ private void registerMXBean() { public boolean isSecurityEnabled() { return UserGroupInformation.isSecurityEnabled(); } + + @VisibleForTesting + static void initFSArgumentHandler(FSConfigToCSConfigConverter converter) { + ResourceManager.fsConfigConversionArgumentHandler = + new FSConfigToCSConfigArgumentHandler(converter); + } + + private static FSConfigToCSConfigConverter initFSConfigConverter() { + FSConfigToCSConfigRuleHandler ruleHandler = + new FSConfigToCSConfigRuleHandler(); + return new FSConfigToCSConfigConverter(ruleHandler); + } + } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/AbstractYarnScheduler.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/AbstractYarnScheduler.java index fe7379a5abe..d7426833ac8 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/AbstractYarnScheduler.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/AbstractYarnScheduler.java @@ -96,6 +96,7 @@ import org.apache.hadoop.yarn.server.resourcemanager.scheduler.event.ContainerPreemptEvent; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.event.ReleaseContainerEvent; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.event.SchedulerEventType; +import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.FairSchedulerConfiguration; import org.apache.hadoop.yarn.server.scheduler.OpportunisticContainerContext; import org.apache.hadoop.yarn.server.scheduler.SchedulerRequestKey; import org.apache.hadoop.yarn.server.utils.BuilderUtils; @@ -182,6 +183,8 @@ protected SchedulingMonitorManager schedulingMonitorManager = new SchedulingMonitorManager(); + private boolean migration; + /** * Construct the service. * @@ -197,6 +200,9 @@ public AbstractYarnScheduler(String name) { @Override public void serviceInit(Configuration conf) throws Exception { + migration = + conf.getBoolean(FairSchedulerConfiguration.MIGRATION_MODE, false); + nmExpireInterval = conf.getInt(YarnConfiguration.RM_NM_EXPIRY_INTERVAL_MS, YarnConfiguration.DEFAULT_RM_NM_EXPIRY_INTERVAL_MS); @@ -209,7 +215,10 @@ public void serviceInit(Configuration conf) throws Exception { nodeTracker.setConfiguredMaxAllocationWaitTime( configuredMaximumAllocationWaitTime); maxClusterLevelAppPriority = getMaxPriorityFromConf(conf); - this.releaseCache = new Timer("Pending Container Clear Timer"); + if (!migration) { + this.releaseCache = new Timer("Pending Container Clear Timer"); + } + autoUpdateContainers = conf.getBoolean(YarnConfiguration.RM_AUTO_UPDATE_CONTAINERS, YarnConfiguration.DEFAULT_RM_AUTO_UPDATE_CONTAINERS); @@ -227,11 +236,14 @@ public void serviceInit(Configuration conf) throws Exception { @Override protected void serviceStart() throws Exception { - if (updateThread != null) { - updateThread.start(); + if (!migration) { + if (updateThread != null) { + updateThread.start(); + } + schedulingMonitorManager.startAll(); + createReleaseCache(); } - schedulingMonitorManager.startAll(); - createReleaseCache(); + super.serviceStart(); } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/AllocationConfiguration.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/AllocationConfiguration.java index 7f065211026..ddbe595c86d 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/AllocationConfiguration.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/AllocationConfiguration.java @@ -242,6 +242,10 @@ public int getUserMaxApps(String user) { Integer maxApps = userMaxApps.get(user); return (maxApps == null) ? userMaxAppsDefault : maxApps; } + + public Map getUserMaxApps() { + return userMaxApps; + } @VisibleForTesting int getQueueMaxApps(String queue) { @@ -249,12 +253,24 @@ int getQueueMaxApps(String queue) { return (maxApps == null) ? queueMaxAppsDefault : maxApps; } + public int getQueueMaxAppsDefault() { + return queueMaxAppsDefault; + } + + public int getUserMaxAppsDefault() { + return userMaxAppsDefault; + } + @VisibleForTesting float getQueueMaxAMShare(String queue) { Float maxAMShare = queueMaxAMShares.get(queue); return (maxAMShare == null) ? queueMaxAMShareDefault : maxAMShare; } + public float getQueueMaxAMShareDefault() { + return queueMaxAMShareDefault; + } + /** * Get the minimum resource allocation for the given queue. * diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/ConfigurableResource.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/ConfigurableResource.java index 44049fdf595..51fc0b7e313 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/ConfigurableResource.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/ConfigurableResource.java @@ -137,4 +137,8 @@ void setPercentage(String name, double value) { } } } + + public double[] getPercentages() { + return percentages; + } } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/ConversionException.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/ConversionException.java new file mode 100644 index 00000000000..7483148b53b --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/ConversionException.java @@ -0,0 +1,37 @@ +/** +* Licensed to the Apache Software Foundation (ASF) under one +* or more contributor license agreements. See the NOTICE file +* distributed with this work for additional information +* regarding copyright ownership. The ASF licenses this file +* to you under the Apache License, Version 2.0 (the +* "License"); you may not use this file except in compliance +* with the License. You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, software +* distributed under the License is distributed on an "AS IS" BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions and +* limitations under the License. +*/ + +package org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair; + +/** + * Thrown when the FS-to-CS converter logic encounters a + * condition from which it cannot recover (eg. unsupported + * settings). + * + */ +public class ConversionException extends RuntimeException { + private static final long serialVersionUID = 4161836727287317835L; + + public ConversionException(String message) { + super(message); + } + + public ConversionException(String message, Throwable cause) { + super(message, cause); + } +} diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSQueue.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSQueue.java index c22fdb05c22..cca08756184 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSQueue.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSQueue.java @@ -184,6 +184,10 @@ public Resource getMaxShare() { return result; } + public ConfigurableResource getRawMaxShare() { + return maxShare; + } + public Resource getReservedResource() { reservedResource.setMemorySize(metrics.getReservedMB()); reservedResource.setVirtualCores(metrics.getReservedVirtualCores()); @@ -207,7 +211,7 @@ public int getMaxRunningApps() { } @VisibleForTesting - protected float getMaxAMShare() { + public float getMaxAMShare() { return maxAMShare; } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FairScheduler.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FairScheduler.java index 151a7ab0867..04bbe0ff684 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FairScheduler.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FairScheduler.java @@ -213,6 +213,8 @@ @VisibleForTesting Resource reservationThreshold; + private boolean migration; + public FairScheduler() { super(FairScheduler.class.getName()); context = new FSContext(this); @@ -1428,7 +1430,7 @@ private void initScheduler(Configuration conf) throws IOException { allocConf = new AllocationConfiguration(this); queueMgr.initialize(); - if (continuousSchedulingEnabled) { + if (continuousSchedulingEnabled && !migration) { // Continuous scheduling is deprecated log it on startup LOG.warn("Continuous scheduling is turned ON. It is deprecated " + "because it can cause scheduler slowness due to locking issues. " + @@ -1441,7 +1443,7 @@ private void initScheduler(Configuration conf) throws IOException { schedulingThread.setDaemon(true); } - if (this.conf.getPreemptionEnabled()) { + if (this.conf.getPreemptionEnabled() && !migration) { createPreemptionThread(); } } finally { @@ -1495,11 +1497,15 @@ private void startSchedulerThreads() { @Override public void serviceInit(Configuration conf) throws Exception { + migration = + conf.getBoolean(FairSchedulerConfiguration.MIGRATION_MODE, false); initScheduler(conf); super.serviceInit(conf); - // Initialize SchedulingMonitorManager - schedulingMonitorManager.initialize(rmContext, conf); + if (!migration) { + // Initialize SchedulingMonitorManager + schedulingMonitorManager.initialize(rmContext, conf); + } } @Override diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FairSchedulerConfiguration.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FairSchedulerConfiguration.java index cfe07c9ef48..1fcb8bc8458 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FairSchedulerConfiguration.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FairSchedulerConfiguration.java @@ -85,29 +85,31 @@ private static final String CONF_PREFIX = "yarn.scheduler.fair."; + public static final String MIGRATION_MODE = CONF_PREFIX + "migration.mode"; + public static final String ALLOCATION_FILE = CONF_PREFIX + "allocation.file"; protected static final String DEFAULT_ALLOCATION_FILE = "fair-scheduler.xml"; /** Whether pools can be created that were not specified in the FS configuration file */ - protected static final String ALLOW_UNDECLARED_POOLS = CONF_PREFIX + "allow-undeclared-pools"; - protected static final boolean DEFAULT_ALLOW_UNDECLARED_POOLS = true; + public static final String ALLOW_UNDECLARED_POOLS = CONF_PREFIX + "allow-undeclared-pools"; + public static final boolean DEFAULT_ALLOW_UNDECLARED_POOLS = true; /** Whether to use the user name as the queue name (instead of "default") if * the request does not specify a queue. */ - protected static final String USER_AS_DEFAULT_QUEUE = CONF_PREFIX + "user-as-default-queue"; - protected static final boolean DEFAULT_USER_AS_DEFAULT_QUEUE = true; + public static final String USER_AS_DEFAULT_QUEUE = CONF_PREFIX + "user-as-default-queue"; + public static final boolean DEFAULT_USER_AS_DEFAULT_QUEUE = true; protected static final float DEFAULT_LOCALITY_THRESHOLD = -1.0f; /** Cluster threshold for node locality. */ - protected static final String LOCALITY_THRESHOLD_NODE = CONF_PREFIX + "locality.threshold.node"; - protected static final float DEFAULT_LOCALITY_THRESHOLD_NODE = + public static final String LOCALITY_THRESHOLD_NODE = CONF_PREFIX + "locality.threshold.node"; + public static final float DEFAULT_LOCALITY_THRESHOLD_NODE = DEFAULT_LOCALITY_THRESHOLD; /** Cluster threshold for rack locality. */ - protected static final String LOCALITY_THRESHOLD_RACK = CONF_PREFIX + "locality.threshold.rack"; - protected static final float DEFAULT_LOCALITY_THRESHOLD_RACK = + public static final String LOCALITY_THRESHOLD_RACK = CONF_PREFIX + "locality.threshold.rack"; + public static final float DEFAULT_LOCALITY_THRESHOLD_RACK = DEFAULT_LOCALITY_THRESHOLD; /** @@ -139,10 +141,10 @@ * {@link #ASSIGN_MULTIPLE} to improve container allocation ramp up. */ @Deprecated - protected static final String CONTINUOUS_SCHEDULING_ENABLED = CONF_PREFIX + + public static final String CONTINUOUS_SCHEDULING_ENABLED = CONF_PREFIX + "continuous-scheduling-enabled"; @Deprecated - protected static final boolean DEFAULT_CONTINUOUS_SCHEDULING_ENABLED = false; + public static final boolean DEFAULT_CONTINUOUS_SCHEDULING_ENABLED = false; /** * Sleep time of each pass in continuous scheduling (5ms in default). @@ -150,21 +152,21 @@ * Only used when {@link #CONTINUOUS_SCHEDULING_ENABLED} is enabled */ @Deprecated - protected static final String CONTINUOUS_SCHEDULING_SLEEP_MS = CONF_PREFIX + + public static final String CONTINUOUS_SCHEDULING_SLEEP_MS = CONF_PREFIX + "continuous-scheduling-sleep-ms"; @Deprecated - protected static final int DEFAULT_CONTINUOUS_SCHEDULING_SLEEP_MS = 5; + public static final int DEFAULT_CONTINUOUS_SCHEDULING_SLEEP_MS = 5; /** Whether preemption is enabled. */ - protected static final String PREEMPTION = CONF_PREFIX + "preemption"; - protected static final boolean DEFAULT_PREEMPTION = false; + public static final String PREEMPTION = CONF_PREFIX + "preemption"; + public static final boolean DEFAULT_PREEMPTION = false; protected static final String PREEMPTION_THRESHOLD = CONF_PREFIX + "preemption.cluster-utilization-threshold"; protected static final float DEFAULT_PREEMPTION_THRESHOLD = 0.8f; - protected static final String WAIT_TIME_BEFORE_KILL = CONF_PREFIX + "waitTimeBeforeKill"; - protected static final int DEFAULT_WAIT_TIME_BEFORE_KILL = 15000; + public static final String WAIT_TIME_BEFORE_KILL = CONF_PREFIX + "waitTimeBeforeKill"; + public static final int DEFAULT_WAIT_TIME_BEFORE_KILL = 15000; /** * Postfix for resource allocation increments in the @@ -181,18 +183,18 @@ * This is intended to be a backdoor on production clusters, and hence * intentionally not documented. */ - protected static final String WAIT_TIME_BEFORE_NEXT_STARVATION_CHECK_MS = + public static final String WAIT_TIME_BEFORE_NEXT_STARVATION_CHECK_MS = CONF_PREFIX + "waitTimeBeforeNextStarvationCheck"; - protected static final long + public static final long DEFAULT_WAIT_TIME_BEFORE_NEXT_STARVATION_CHECK_MS = 10000; /** Whether to assign multiple containers in one check-in. */ public static final String ASSIGN_MULTIPLE = CONF_PREFIX + "assignmultiple"; - protected static final boolean DEFAULT_ASSIGN_MULTIPLE = false; + public static final boolean DEFAULT_ASSIGN_MULTIPLE = false; /** Whether to give more weight to apps requiring many resources. */ - protected static final String SIZE_BASED_WEIGHT = CONF_PREFIX + "sizebasedweight"; - protected static final boolean DEFAULT_SIZE_BASED_WEIGHT = false; + public static final String SIZE_BASED_WEIGHT = CONF_PREFIX + "sizebasedweight"; + public static final boolean DEFAULT_SIZE_BASED_WEIGHT = false; /** Maximum number of containers to assign on each check-in. */ public static final String DYNAMIC_MAX_ASSIGN = @@ -203,8 +205,8 @@ * Specify exact number of containers to assign on each heartbeat, if dynamic * max assign is turned off. */ - protected static final String MAX_ASSIGN = CONF_PREFIX + "max.assign"; - protected static final int DEFAULT_MAX_ASSIGN = -1; + public static final String MAX_ASSIGN = CONF_PREFIX + "max.assign"; + public static final int DEFAULT_MAX_ASSIGN = -1; /** The update interval for calculating resources in FairScheduler .*/ public static final String UPDATE_INTERVAL_MS = diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/converter/FSConfigToCSConfigArgumentHandler.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/converter/FSConfigToCSConfigArgumentHandler.java new file mode 100644 index 00000000000..57298e21c59 --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/converter/FSConfigToCSConfigArgumentHandler.java @@ -0,0 +1,192 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * http://www.apache.org/licenses/LICENSE-2.0 + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.converter; + +import org.apache.commons.cli.CommandLine; +import org.apache.commons.cli.GnuParser; +import org.apache.commons.cli.MissingArgumentException; +import org.apache.commons.cli.Option; +import org.apache.commons.cli.Options; +import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.ConversionException; + +import java.io.File; + +/** + * Parses arguments passed to the FS->CS converter. + * If the arguments are valid, it calls the converter itself. + * + */ +public class FSConfigToCSConfigArgumentHandler { + private final FSConfigToCSConfigConverter converter; + + public FSConfigToCSConfigArgumentHandler(FSConfigToCSConfigConverter + converter) { + this.converter = converter; + } + + public enum CliOption { + YARN_SITE("yarn-site.xml", "y", "yarnsiteconfig", + "Path to a valid yarn-site.xml config file", true, true), + + // fair-scheduler.xml is not mandatory + // if FairSchedulerConfiguration.ALLOCATION_FILE is defined in yarn-site.xml + FAIR_SCHEDULER("fair-scheduler.xml", "f", "fsconfig", + "Path to a valid fair-scheduler.xml config file", false, true), + CONVERSION_RULES("conversion rules config file", "r", "rulesconfig", + "Optional parameter. If given, should specify a valid path to the " + + "conversion rules file (property format).", false, true), + CONSOLE_MODE("console mode", "p", "print", + "If defined, the converted configuration will " + + "only be emitted to the console.", false, false), + CLUSTER_RESOURCE("cluster resource", "c", "cluster-resource", + "Needs to be given if maxResources is defined as percentages " + + "for any queue, otherwise this parameter can be omitted.", + false, true), + OUTPUT_DIR("output directory", "o", "output-directory", + "Output directory for yarn-site.xml and" + + " capacity-scheduler.xml files." + + "Must have write permission for user who is running this script.", + true, true); + + private final String name; + private final String shortSwitch; + private final String longSwitch; + private final String description; + private final boolean required; + private final boolean hasArg; + + CliOption(String name, String shortSwitch, String longSwitch, + String description, boolean required, boolean hasArg) { + this.name = name; + this.shortSwitch = shortSwitch; + this.longSwitch = longSwitch; + this.description = description; + this.required = required; + this.hasArg = hasArg; + } + + public Option createCommonsCliOption() { + Option option = new Option(shortSwitch, longSwitch, hasArg, description); + option.setRequired(required); + return option; + } + + public String getAsArgumentString() { + return shortSwitch + "|" + longSwitch + ": " + description; + } + } + + public void parseAndConvert(String[] args) throws Exception { + Options opts = createOptions(); + CommandLine cliParser; + try { + cliParser = new GnuParser().parse(opts, args); + } catch (MissingArgumentException ex) { + throw new IllegalArgumentException("Missing argument for options", ex); + } + + checkOptionPresent(cliParser, CliOption.YARN_SITE); + checkOptionPresent(cliParser, CliOption.OUTPUT_DIR); + FSConfigToCSConfigConverterParams params = validateInputFiles(cliParser); + try { + converter.convert(params); + } catch (UnsupportedPropertyException e) { + System.err.println( + "Fatal error during conversion! " + e.getMessage()); + } catch (ConversionException e) { + System.err.println( + "Error occurred during property conversion: " + e.getMessage()); + } + } + + private Options createOptions() { + Options opts = new Options(); + + for (CliOption cliOption : CliOption.values()) { + opts.addOption(cliOption.createCommonsCliOption()); + } + + return opts; + } + + private FSConfigToCSConfigConverterParams validateInputFiles( + CommandLine cliParser) { + String yarnSiteXmlFile = + cliParser.getOptionValue(CliOption.YARN_SITE.shortSwitch); + String fairSchedulerXmlFile = + cliParser.getOptionValue(CliOption.FAIR_SCHEDULER.shortSwitch); + String conversionRulesFile = + cliParser.getOptionValue(CliOption.CONVERSION_RULES.shortSwitch); + String outputDir = + cliParser.getOptionValue(CliOption.OUTPUT_DIR.shortSwitch); + + checkFile(CliOption.YARN_SITE, yarnSiteXmlFile); + checkFile(CliOption.FAIR_SCHEDULER, fairSchedulerXmlFile); + checkFile(CliOption.CONVERSION_RULES, conversionRulesFile); + checkDirectory(CliOption.OUTPUT_DIR, outputDir); + + return FSConfigToCSConfigConverterParams.Builder.create() + .withYarnSiteXmlConfig(yarnSiteXmlFile) + .withFairSchedulerXmlConfig(fairSchedulerXmlFile) + .withConversionRulesConfig(conversionRulesFile) + .withClusterResource( + cliParser.getOptionValue(CliOption.CLUSTER_RESOURCE.shortSwitch)) + .withConsole(cliParser.hasOption(CliOption.CONSOLE_MODE.shortSwitch)) + .withOutputDirectory(outputDir) + .build(); + } + + private static void checkOptionPresent(CommandLine cliParser, + CliOption cliOption) { + if (!cliParser.hasOption(cliOption.shortSwitch)) { + throw new IllegalArgumentException( + String.format("Missing %s parameter " + "(switch: %s|%s).", + cliOption.name, cliOption.shortSwitch, cliOption.longSwitch)); + } + } + + private static void checkFile(CliOption cliOption, String filePath) { + checkFileInternal(cliOption, filePath, true); + } + + private static void checkDirectory(CliOption cliOption, String dirPath) { + checkFileInternal(cliOption, dirPath, false); + } + + private static void checkFileInternal(CliOption cliOption, String filePath, + boolean isFile) { + //We can safely ignore null here as files / dirs were checked before + if (filePath == null) { + return; + } + + File file = new File(filePath); + if (isFile && file.isDirectory()) { + throw new IllegalArgumentException( + String.format("Specified path %s is a directory but should be " + + " a file (As value of parameter %s)", filePath, cliOption.name)); + } else if (!isFile && !file.isDirectory()) { + throw new IllegalArgumentException( + String.format("Specified path %s is not a directory " + + "(As value of parameter %s)", filePath, cliOption.name)); + } else if (!file.exists()) { + throw new IllegalArgumentException( + String.format("Specified path %s does not exist " + + "(As value of parameter %s)", filePath, cliOption.name)); + } + } +} diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/converter/FSConfigToCSConfigConverter.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/converter/FSConfigToCSConfigConverter.java new file mode 100644 index 00000000000..6c42357b4d7 --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/converter/FSConfigToCSConfigConverter.java @@ -0,0 +1,365 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * http://www.apache.org/licenses/LICENSE-2.0 + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.converter; + +import static org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacitySchedulerConfiguration.PREFIX; + +import java.io.File; +import java.io.FileNotFoundException; +import java.io.FileOutputStream; +import java.io.IOException; +import java.io.OutputStream; +import java.util.Map; + +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.Path; +import org.apache.hadoop.security.authorize.AccessControlList; +import org.apache.hadoop.yarn.api.records.Resource; +import org.apache.hadoop.yarn.conf.YarnConfiguration; +import org.apache.hadoop.yarn.security.AccessType; +import org.apache.hadoop.yarn.server.resourcemanager.RMContext; +import org.apache.hadoop.yarn.server.resourcemanager.RMContextImpl; +import org.apache.hadoop.yarn.server.resourcemanager.placement.PlacementManager; +import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacitySchedulerConfiguration; +import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.AllocationConfiguration; +import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.AllocationConfigurationException; +import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.ConfigurableResource; +import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.ConversionException; +import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.FSParentQueue; +import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.FairScheduler; +import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.FairSchedulerConfiguration; +import org.apache.hadoop.yarn.util.resource.DominantResourceCalculator; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import com.google.common.annotations.VisibleForTesting; + +/** + * Converts Fair Scheduler configuration (site and fair-scheduler.xml) + * to Capacity Scheduler. The mapping is not 100% perfect due to + * feature gaps. These will be addressed in the future. + */ +public class FSConfigToCSConfigConverter { + public static final Logger LOG = LoggerFactory.getLogger( + FSConfigToCSConfigConverter.class.getName()); + + private Resource clusterResource; + private boolean preemptionEnabled = false; + private int queueMaxAppsDefault; + private float queueMaxAMShareDefault; + private boolean autoCreateChildQueues = false; + private boolean sizeBasedWeight = false; + private boolean userAsDefaultQueue = false; + + private Configuration yarnSiteConfig; + private Configuration capacitySchedulerConfig; + private FSConfigToCSConfigRuleHandler ruleHandler; + + private OutputStream yarnSiteOutputStream; + private OutputStream capacitySchedulerOutputStream; + private boolean consoleMode = false; + + private enum OutputFile { + YARN_SITE_XML("yarn-site.xml"), + CAPACITY_SCHEDULER_XML("capacity-scheduler.xml"); + + private final String name; + + OutputFile(String name) { + this.name = name; + } + } + + public FSConfigToCSConfigConverter(FSConfigToCSConfigRuleHandler + ruleHandler) { + this.ruleHandler = ruleHandler; + this.yarnSiteOutputStream = System.out; + this.capacitySchedulerOutputStream = System.out; + } + + public void convert(FSConfigToCSConfigConverterParams params) + throws Exception { + validateParams(params); + prepareOutputFiles(params.getOutputDirectory(), params.isConsole()); + loadConversionRules(params.getConversionRulesConfig()); + Configuration conf = createConfiguration(params); + handleFairSchedulerConfig(params, conf); + + convert(conf, getClusterResource(params)); + } + + private void prepareOutputFiles(String outputDirectory, boolean console) + throws FileNotFoundException { + if (console) { + LOG.info("Console mode is enabled, yarn-site.xml and" + + " capacity-scheduler.xml will be only emitted to the console!"); + this.consoleMode = true; + return; + } + File yarnSiteXmlOutputFile = new File(outputDirectory, + OutputFile.YARN_SITE_XML.name); + File schedulerXmlOutput = new File(outputDirectory, + OutputFile.CAPACITY_SCHEDULER_XML.name); + LOG.info("Output directory for yarn-site.xml and" + + " capacity-scheduler.xml is: {}", outputDirectory); + + this.yarnSiteOutputStream = new FileOutputStream(yarnSiteXmlOutputFile); + this.capacitySchedulerOutputStream = + new FileOutputStream(schedulerXmlOutput); + } + + private void validateParams(FSConfigToCSConfigConverterParams params) { + if (params.getYarnSiteXmlConfig() == null) { + throw new IllegalArgumentException("yarn-site.xml configuration " + + "is not defined but it is mandatory!"); + } else if (params.getOutputDirectory() == null && !params.isConsole()) { + throw new IllegalArgumentException("Output directory configuration " + + "is not defined but it is mandatory!"); + } + } + + private Resource getClusterResource( + FSConfigToCSConfigConverterParams params) { + Resource resource = null; + if (params.getClusterResource() != null) { + ConfigurableResource configurableResource; + try { + configurableResource = FairSchedulerConfiguration + .parseResourceConfigValue(params.getClusterResource()); + } catch (AllocationConfigurationException e) { + throw new ConversionException("Error while parsing resource.", e); + } + resource = configurableResource.getResource(); + } + return resource; + } + + private void loadConversionRules(String rulesFile) throws IOException { + if (rulesFile != null) { + LOG.info("Reading conversion rules file from: " + rulesFile); + this.ruleHandler.loadRulesFromFile(rulesFile); + } else { + LOG.info("Conversion rules file is not defined, " + + "using default conversion config!"); + } + } + + private Configuration createConfiguration( + FSConfigToCSConfigConverterParams params) { + Configuration conf = new YarnConfiguration(); + conf.addResource(new Path(params.getYarnSiteXmlConfig())); + conf.setBoolean(FairSchedulerConfiguration.MIGRATION_MODE, true); + return conf; + } + + private void handleFairSchedulerConfig( + FSConfigToCSConfigConverterParams params, Configuration conf) { + String fairSchedulerXmlConfig = params.getFairSchedulerXmlConfig(); + + // Don't override allocation file in conf yet, as it would ruin the second + // condition here + if (fairSchedulerXmlConfig != null) { + LOG.info("Using explicitly defined fair-scheduler.xml"); + } else if (conf.get(FairSchedulerConfiguration.ALLOCATION_FILE) != null) { + LOG.info("Using fair-scheduler.xml defined in yarn-site.xml by key: " + + FairSchedulerConfiguration.ALLOCATION_FILE); + } else { + throw new IllegalArgumentException("fair-scheduler.xml is not defined " + + "neither in yarn-site.xml " + + "(with property: " + FairSchedulerConfiguration.ALLOCATION_FILE + + ") nor directly with its own parameter!"); + } + + // We can now safely override allocation file in conf + if (fairSchedulerXmlConfig != null) { + conf.set(FairSchedulerConfiguration.ALLOCATION_FILE, + params.getFairSchedulerXmlConfig()); + } + } + + @VisibleForTesting + void convert(Configuration conf, Resource clusterResource) throws Exception { + // initialize Fair Scheduler + RMContext ctx = new RMContextImpl(); + PlacementManager placementManager = new PlacementManager(); + ctx.setQueuePlacementManager(placementManager); + + FairScheduler fs = new FairScheduler(); + fs.setRMContext(ctx); + fs.init(conf); + this.clusterResource = clusterResource; + + AllocationConfiguration allocConf = fs.getAllocationConfiguration(); + queueMaxAppsDefault = allocConf.getQueueMaxAppsDefault(); + queueMaxAMShareDefault = allocConf.getQueueMaxAMShareDefault(); + + yarnSiteConfig = new Configuration(false); + capacitySchedulerConfig = new Configuration(false); + + checkUserMaxApps(allocConf); + checkUserMaxAppsDefault(allocConf); + + convertYarnSiteXml(conf); + convertCapacitySchedulerXml(fs); + + if (consoleMode) { + System.out.println("======= capacity-scheduler.xml ======="); + } + capacitySchedulerConfig.writeXml(capacitySchedulerOutputStream); + + if (consoleMode) { + System.out.println(); + System.out.println("======= yarn-site.xml ======="); + } + yarnSiteConfig.writeXml(yarnSiteOutputStream); + } + + @VisibleForTesting + void setYarnSiteOutputStream(OutputStream out) { + this.yarnSiteOutputStream = out; + } + + @VisibleForTesting + void setCapacitySchedulerConfigOutputStream(OutputStream out) { + this.capacitySchedulerOutputStream = out; + } + + private void convertYarnSiteXml(Configuration conf) { + FSYarnSiteConverter siteConverter = + new FSYarnSiteConverter(); + siteConverter.convertSiteProperties(conf, yarnSiteConfig); + + autoCreateChildQueues = siteConverter.isAutoCreateChildQueues(); + preemptionEnabled = siteConverter.isPreemptionEnabled(); + sizeBasedWeight = siteConverter.isSizeBasedWeight(); + userAsDefaultQueue = siteConverter.isUserAsDefaultQueue(); + + checkReservationSystem(conf); + } + + private void convertCapacitySchedulerXml(FairScheduler fs) { + FSParentQueue rootQueue = fs.getQueueManager().getRootQueue(); + emitDefaultMaxApplications(); + emitDefaultMaxAMshare(); + FSQueueConverter queueConverter = new FSQueueConverter(ruleHandler, + capacitySchedulerConfig, + preemptionEnabled, + sizeBasedWeight, + autoCreateChildQueues, + clusterResource, + queueMaxAMShareDefault, + queueMaxAppsDefault); + queueConverter.convertQueue(rootQueue); + emitACLs(fs); + + PlacementManager placementManager = + fs.getRMContext().getQueuePlacementManager(); + + if (placementManager.getPlacementRules().size() > 0) { + QueuePlacementConverter placementConverter = + new QueuePlacementConverter(); + Map properties = + placementConverter.convertPlacementPolicy(placementManager, + ruleHandler, userAsDefaultQueue); + properties.entrySet().forEach(entry -> + capacitySchedulerConfig.set(entry.getKey(), entry.getValue())); + } + + // Validate ordering policy + if (queueConverter.isDrfPolicyUsedOnQueueLevel()) { + if (queueConverter.isFifoOrFairSharePolicyUsed()) { + throw new ConversionException( + "DRF ordering policy cannot be used together with fifo/fair"); + } else { + capacitySchedulerConfig.set( + CapacitySchedulerConfiguration.RESOURCE_CALCULATOR_CLASS, + DominantResourceCalculator.class.getCanonicalName()); + } + } + } + + private void emitDefaultMaxApplications() { + if (queueMaxAppsDefault != Integer.MAX_VALUE) { + capacitySchedulerConfig.set( + CapacitySchedulerConfiguration.MAXIMUM_SYSTEM_APPLICATIONS, + String.valueOf(queueMaxAppsDefault)); + } + } + + private void emitDefaultMaxAMshare() { + capacitySchedulerConfig.set( + CapacitySchedulerConfiguration. + MAXIMUM_APPLICATION_MASTERS_RESOURCE_PERCENT, + String.valueOf(queueMaxAMShareDefault)); + } + + private void emitACLs(FairScheduler fs) { + fs.getAllocationConfiguration().getQueueAcls() + .entrySet() + .stream() + .forEach(entry -> generateQueueAcl(entry.getKey(), + entry.getValue())); + } + + private void generateQueueAcl(String queue, + Map access) { + AccessControlList submitAcls = access.get(AccessType.SUBMIT_APP); + AccessControlList adminAcls = access.get(AccessType.ADMINISTER_QUEUE); + + if (!submitAcls.getGroups().isEmpty() || + !submitAcls.getUsers().isEmpty()) { + capacitySchedulerConfig.set(PREFIX + queue + ".acl_submit_applications", + submitAcls.getAclString()); + } + + if (!adminAcls.getGroups().isEmpty() || + !adminAcls.getUsers().isEmpty()) { + capacitySchedulerConfig.set(PREFIX + queue + ".acl_administer_queue", + adminAcls.getAclString()); + } + } + + private void checkReservationSystem(Configuration conf) { + if (conf.getBoolean(YarnConfiguration.RM_RESERVATION_SYSTEM_ENABLE, + YarnConfiguration.DEFAULT_RM_RESERVATION_SYSTEM_ENABLE)) { + ruleHandler.handleReservationSystem(); + } + } + + private void checkUserMaxApps(AllocationConfiguration allocConf) { + if (allocConf.getUserMaxApps() != null + && allocConf.getUserMaxApps().size() > 0) { + ruleHandler.handleUserMaxApps(); + } + } + + private void checkUserMaxAppsDefault(AllocationConfiguration allocConf) { + if (allocConf.getUserMaxAppsDefault() > 0) { + ruleHandler.handleUserMaxAppsDefault(); + } + } + + @VisibleForTesting + Resource getClusterResource() { + return clusterResource; + } + + @VisibleForTesting + FSConfigToCSConfigRuleHandler getRuleHandler() { + return ruleHandler; + } +} diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/converter/FSConfigToCSConfigConverterParams.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/converter/FSConfigToCSConfigConverterParams.java new file mode 100644 index 00000000000..b083b252d4c --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/converter/FSConfigToCSConfigConverterParams.java @@ -0,0 +1,131 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * http://www.apache.org/licenses/LICENSE-2.0 + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.converter; + +/** + * POJO that holds values for the FS->CS converter. + * + */ +public final class FSConfigToCSConfigConverterParams { + private String yarnSiteXmlConfig; + private String fairSchedulerXmlConfig; + private String conversionRulesConfig; + private boolean console; + private String clusterResource; + private String outputDirectory; + + private FSConfigToCSConfigConverterParams() { + //must use builder + } + + public String getFairSchedulerXmlConfig() { + return fairSchedulerXmlConfig; + } + + public String getYarnSiteXmlConfig() { + return yarnSiteXmlConfig; + } + + public String getConversionRulesConfig() { + return conversionRulesConfig; + } + + public String getClusterResource() { + return clusterResource; + } + + public boolean isConsole() { + return console; + } + + public String getOutputDirectory() { + return outputDirectory; + } + + @Override + public String toString() { + return "FSConfigToCSConfigConverterParams{" + + "yarnSiteXmlConfig='" + yarnSiteXmlConfig + '\'' + + ", fairSchedulerXmlConfig='" + fairSchedulerXmlConfig + '\'' + + ", conversionRulesConfig='" + conversionRulesConfig + '\'' + + ", clusterResource='" + clusterResource + '\'' + + ", console=" + console + + '}'; + } + + /** + * Builder that can construct FSConfigToCSConfigConverterParams objects. + * + */ + public static final class Builder { + private String yarnSiteXmlConfig; + private String fairSchedulerXmlConfig; + private String conversionRulesConfig; + private boolean console; + private String clusterResource; + private String outputDirectory; + + private Builder() { + } + + public static Builder create() { + return new Builder(); + } + + public Builder withYarnSiteXmlConfig(String config) { + this.yarnSiteXmlConfig = config; + return this; + } + + public Builder withFairSchedulerXmlConfig(String config) { + this.fairSchedulerXmlConfig = config; + return this; + } + + public Builder withConversionRulesConfig(String config) { + this.conversionRulesConfig = config; + return this; + } + + public Builder withClusterResource(String res) { + this.clusterResource = res; + return this; + } + + public Builder withConsole(boolean console) { + this.console = console; + return this; + } + + public Builder withOutputDirectory(String outputDir) { + this.outputDirectory = outputDir; + return this; + } + + public FSConfigToCSConfigConverterParams build() { + FSConfigToCSConfigConverterParams params = + new FSConfigToCSConfigConverterParams(); + params.clusterResource = this.clusterResource; + params.console = this.console; + params.fairSchedulerXmlConfig = this.fairSchedulerXmlConfig; + params.yarnSiteXmlConfig = this.yarnSiteXmlConfig; + params.conversionRulesConfig = this.conversionRulesConfig; + params.outputDirectory = this.outputDirectory; + return params; + } + } +} diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/converter/FSConfigToCSConfigRuleHandler.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/converter/FSConfigToCSConfigRuleHandler.java new file mode 100644 index 00000000000..26a0e1b987e --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/converter/FSConfigToCSConfigRuleHandler.java @@ -0,0 +1,227 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * http://www.apache.org/licenses/LICENSE-2.0 + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.converter; + +import static java.lang.String.format; + +import java.io.File; +import java.io.FileInputStream; +import java.io.IOException; +import java.util.HashMap; +import java.util.Map; +import java.util.Properties; + +import org.apache.commons.lang3.StringUtils; + +import com.google.common.annotations.VisibleForTesting; +import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.ConversionException; +import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.FairSchedulerConfiguration; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +/** + * Class that determines what should happen if the FS->CS converter + * encounters a property that is currently not supported. + * + * Acceptable values are either "abort" or "warning". + */ +public class FSConfigToCSConfigRuleHandler { + private static final Logger LOG = + LoggerFactory.getLogger(FSConfigToCSConfigRuleHandler.class); + + + public static final String MAX_CHILD_QUEUE_LIMIT = + "maxChildQueue.limit"; + + public static final String MAX_CAPACITY_PERCENTAGE = + "maxCapacityPercentage.action"; + + public static final String MAX_CHILD_CAPACITY = + "maxChildCapacity.action"; + + public static final String USER_MAX_RUNNING_APPS = + "userMaxRunningApps.action"; + + public static final String USER_MAX_APPS_DEFAULT = + "userMaxAppsDefault.action"; + + public static final String DYNAMIC_MAX_ASSIGN = + "dynamicMaxAssign.action"; + + public static final String SPECIFIED_NOT_FIRST = + "specifiedNotFirstRule.action"; + + public static final String RESERVATION_SYSTEM = + "reservationSystem.action"; + + public static final String QUEUE_AUTO_CREATE = + "queueAutoCreate.action"; + + @VisibleForTesting + enum RuleAction { + WARNING, + ABORT + } + + private Map actions; + private Properties properties; + + void loadRulesFromFile(String ruleFile) throws IOException { + if (ruleFile == null) { + throw new IllegalArgumentException("Rule file cannot be null!"); + } + + properties = new Properties(); + File rules = new File(ruleFile); + properties.load(new FileInputStream(rules)); + actions = new HashMap<>(); + initPropertyActions(); + } + + public FSConfigToCSConfigRuleHandler() { + properties = new Properties(); + actions = new HashMap<>(); + } + + @VisibleForTesting + FSConfigToCSConfigRuleHandler(Properties props) { + properties = props; + actions = new HashMap<>(); + initPropertyActions(); + } + + private void initPropertyActions() { + setActionForProperty(MAX_CAPACITY_PERCENTAGE); + setActionForProperty(MAX_CHILD_CAPACITY); + setActionForProperty(USER_MAX_RUNNING_APPS); + setActionForProperty(USER_MAX_APPS_DEFAULT); + setActionForProperty(DYNAMIC_MAX_ASSIGN); + setActionForProperty(SPECIFIED_NOT_FIRST); + setActionForProperty(RESERVATION_SYSTEM); + setActionForProperty(QUEUE_AUTO_CREATE); + } + + public void handleMaxCapacityPercentage(String queueName) { + handle(MAX_CAPACITY_PERCENTAGE, null, + format(" defined in percentages for queue %s", + queueName)); + } + + public void handleMaxChildCapacity() { + handle(MAX_CHILD_CAPACITY, "", null); + } + + public void handleChildQueueCount(String queue, int count) { + String value = properties.getProperty(MAX_CHILD_QUEUE_LIMIT); + if (value != null) { + if (StringUtils.isNumeric(value)) { + int maxChildQueue = Integer.parseInt(value); + if (count > maxChildQueue) { + throw new ConversionException( + format("Queue %s has too many children: %d", queue, count)); + } + } else { + throw new IllegalArgumentException( + "maxChildQueue.limit is not an integer"); + } + } + } + + public void handleUserMaxApps() { + handle(USER_MAX_RUNNING_APPS, "", null); + } + + public void handleUserMaxAppsDefault() { + handle(USER_MAX_APPS_DEFAULT, "", null); + } + + public void handleDynamicMaxAssign() { + handle(DYNAMIC_MAX_ASSIGN, + FairSchedulerConfiguration.DYNAMIC_MAX_ASSIGN, null); + } + + public void handleSpecifiedNotFirstRule() { + handle(SPECIFIED_NOT_FIRST, + null, + "The tag is not the first placement rule, this cannot be" + + " converted properly"); + } + + public void handleReservationSystem() { + handle(RESERVATION_SYSTEM, + null, + "Conversion of reservation system is not supported"); + } + + public void handleQueueAutoCreate(String placementRule) { + handle(QUEUE_AUTO_CREATE, + null, + format( + "Placement rules: queue auto-create is not supported (type: %s)", + placementRule)); + } + + private void handle(String actionName, String fsSetting, String message) { + RuleAction action = actions.get(actionName); + + if (action != null) { + switch (action) { + case ABORT: + String exceptionMessage; + if (message != null) { + exceptionMessage = message; + } else { + exceptionMessage = format("Setting %s is not supported", fsSetting); + } + throw new UnsupportedPropertyException(exceptionMessage); + case WARNING: + if (message != null) { + LOG.warn(message); + } else { + LOG.warn("Setting {} is not supported, ignoring conversion", + fsSetting); + } + break; + default: + throw new IllegalArgumentException( + "Unknown action " + action); + } + } + } + + private void setActionForProperty(String property) { + String action = properties.getProperty(property); + + if (action == null) { + LOG.info("No rule set for {}, defaulting to WARNING", property); + actions.put(property, RuleAction.WARNING); + } else if (action.equalsIgnoreCase("warning")) { + actions.put(property, RuleAction.WARNING); + } else if (action.equalsIgnoreCase("abort")) { + actions.put(property, RuleAction.ABORT); + } else { + LOG.warn("Unknown action {} set for rule {}, defaulting to WARNING", + action, property); + actions.put(property, RuleAction.WARNING); + } + } + + @VisibleForTesting + public Map getActions() { + return actions; + } +} diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/converter/FSQueueConverter.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/converter/FSQueueConverter.java new file mode 100644 index 00000000000..886a54c401f --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/converter/FSQueueConverter.java @@ -0,0 +1,454 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * http://www.apache.org/licenses/LICENSE-2.0 + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.converter; + +import static org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacitySchedulerConfiguration.PREFIX; + +import java.math.BigDecimal; +import java.math.RoundingMode; +import java.util.HashMap; +import java.util.HashSet; +import java.util.List; +import java.util.Map; +import java.util.Set; +import java.util.stream.Collectors; + +import org.apache.commons.lang3.mutable.MutableBoolean; +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.yarn.api.records.Resource; +import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.ConfigurableResource; +import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.ConversionException; +import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.FSLeafQueue; +import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.FSQueue; +import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.policies.DominantResourceFairnessPolicy; +import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.policies.FairSharePolicy; +import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.policies.FifoPolicy; +import org.apache.hadoop.yarn.util.resource.ResourceUtils; +import org.apache.hadoop.yarn.util.resource.Resources; + +/** + * Converts a Fair Schedule queue hierarchy to Capacity Scheduler + * configuration. + * + */ +public class FSQueueConverter { + private final Set leafQueueNames; + private final FSConfigToCSConfigRuleHandler ruleHandler; + private Configuration capacitySchedulerConfig; + private final boolean preemptionEnabled; + private final boolean sizeBasedWeight; + private final Resource clusterResource; + private final float queueMaxAMShareDefault; + private final boolean autoCreateChildQueues; + private final int queueMaxAppsDefault; + + private boolean fifoOrFairSharePolicyUsed; + private boolean drfPolicyUsedOnQueueLevel; + + @SuppressWarnings("checkstyle:parameternumber") + public FSQueueConverter(FSConfigToCSConfigRuleHandler ruleHandler, + Configuration capacitySchedulerConfig, + boolean preemptionEnabled, + boolean sizeBasedWeight, + boolean autoCreateChildQueues, + Resource clusterResource, + float queueMaxAMShareDefault, + int queueMaxAppsDefault) { + this.leafQueueNames = new HashSet<>(); + this.ruleHandler = ruleHandler; + this.capacitySchedulerConfig = capacitySchedulerConfig; + this.preemptionEnabled = preemptionEnabled; + this.sizeBasedWeight = sizeBasedWeight; + this.clusterResource = clusterResource; + this.queueMaxAMShareDefault = queueMaxAMShareDefault; + this.autoCreateChildQueues = autoCreateChildQueues; + this.queueMaxAppsDefault = queueMaxAppsDefault; + } + + @SuppressWarnings("checkstyle:linelength") + public void convertQueue(FSQueue queue) { + List children = queue.getChildQueues(); + final String queueName = queue.getName(); + + if (queue instanceof FSLeafQueue) { + String shortName = getQueueShortName(queueName); + if (!leafQueueNames.add(shortName)) { + throw new ConversionException( + "Leaf queues must be unique, " + + shortName + " is defined at least twice"); + } + } + + // generates yarn.scheduler.capacity..queues + emitChildQueues(queueName, children); + + // ==> + // yarn.scheduler.capacity..maximum-am-resource-percent + emitMaxAMShare(queueName, queue); + + + // ==> + // yarn.scheduler.capacity..maximum-applications + emitMaxRunningApps(queueName, queue); + + // ==> + // yarn.scheduler.capacity..maximum-allocation-mb/vcores + emitMaxAllocations(queueName, queue); + + // ==> + // yarn.scheduler.capacity..disable_preemption + emitPreemptionDisabled(queueName, queue); + + // TODO: COULD BE incorrect! Needs further clarifications + // weight + minResources ==> yarn.scheduler.capacity..capacity + emitChildCapacity(queue); + + // ==> yarn.scheduler.capacity..maximum-capacity + emitMaximumCapacity(queueName, queue); + + // yarn.scheduler.fair.allow-undeclared-pools ==> + // yarn.scheduler.capacity..auto-create-child-queue.enabled + emitAutoCreateChildQueue(queueName); + + + // yarn.scheduler.fair.sizebasedweight ==> + // yarn.scheduler.capacity..ordering-policy.fair.enable-size-based-weight + emitSizeBasedWeight(queueName); + + // ==> + // yarn.scheduler.capacity..ordering-policy + emitOrderingPolicy(queueName, queue); + + + // missing feature, "leaf-queue-template.capacity" only accepts a single pct value + checkMaxChildCapacitySetting(queue); + + for (FSQueue childQueue : children) { + convertQueue(childQueue); + } + } + + public boolean isFifoOrFairSharePolicyUsed() { + return fifoOrFairSharePolicyUsed; + } + + public boolean isDrfPolicyUsedOnQueueLevel() { + return drfPolicyUsedOnQueueLevel; + } + + private void emitChildQueues(String queueName, List children) { + ruleHandler.handleChildQueueCount(queueName, children.size()); + + if (children.size() > 0) { + String childQueues = children.stream() + .map(child -> getQueueShortName(child.getName())) + .collect(Collectors.joining(",")); + + capacitySchedulerConfig.set(PREFIX + queueName + ".queues", + childQueues.toString()); + } + } + + private void emitMaxAMShare(String queueName, FSQueue queue) { + float queueMaxAmShare = queue.getMaxAMShare(); + + if (queueMaxAmShare != 0.0f + && queueMaxAmShare != queueMaxAMShareDefault + && queueMaxAmShare != -1.0f) { + capacitySchedulerConfig.set(PREFIX + queueName + + ".maximum-am-resource-percent", String.valueOf(queueMaxAmShare)); + } + + if (queueMaxAmShare == -1.0f) { + capacitySchedulerConfig.set(PREFIX + queueName + + ".maximum-am-resource-percent", "1.0"); + } + } + + private void emitMaxRunningApps(String queueName, FSQueue queue) { + if (queue.getMaxRunningApps() != Integer.MAX_VALUE + && queue.getMaxRunningApps() != queueMaxAppsDefault) { + capacitySchedulerConfig.set(PREFIX + queueName + ".maximum-applications", + String.valueOf(queue.getMaxRunningApps())); + } + } + + private void emitMaximumCapacity(String queueName, FSQueue queue) { + ConfigurableResource rawMaxShare = queue.getRawMaxShare(); + final Resource maxResource = rawMaxShare.getResource(); + + long memSize = 0; + long vCores = 0; + boolean defined = false; + + if (maxResource == null) { + if (rawMaxShare.getPercentages() != null) { + if (clusterResource == null) { + throw new ConversionException( + String.format(" defined in percentages for" + + " queue %s, but cluster resource parameter is not" + + " defined via CLI!", queueName)); + } + + ruleHandler.handleMaxCapacityPercentage(queueName); + + double[] percentages = rawMaxShare.getPercentages(); + int memIndex = ResourceUtils.getResourceTypeIndex().get("memory-mb"); + int vcoreIndex = ResourceUtils.getResourceTypeIndex().get("vcores"); + + memSize = (long) (percentages[memIndex] * + clusterResource.getMemorySize()); + vCores = (long) (percentages[vcoreIndex] * + clusterResource.getVirtualCores()); + defined = true; + } else { + throw new IllegalArgumentException( + "Illegal ConfigurableResource = " + rawMaxShare); + } + } else if (Resources.unbounded().compareTo(maxResource) != 0) { + memSize = maxResource.getMemorySize(); + vCores = maxResource.getVirtualCores(); + defined = true; + } + + if (defined) { + capacitySchedulerConfig.set(PREFIX + queueName + ".maximum-capacity", + String.format("[memory=%d, vcores=%d]", memSize, vCores)); + } + } + + private void emitMaxAllocations(String queueName, FSQueue queue) { + Resource maxAllocation = queue.getMaximumContainerAllocation(); + + if (Resources.unbounded().compareTo(maxAllocation) != 0) { + long parentMaxVcores = Integer.MIN_VALUE; + long parentMaxMemory = Integer.MIN_VALUE; + + if (queue.getParent() != null) { + FSQueue parent = queue.getParent(); + Resource parentMaxAllocation = parent.getMaximumContainerAllocation(); + if (Resources.unbounded().compareTo(parentMaxAllocation) != 0) { + parentMaxVcores = parentMaxAllocation.getVirtualCores(); + parentMaxMemory = parentMaxAllocation.getMemorySize(); + } + } + + long maxVcores = maxAllocation.getVirtualCores(); + long maxMemory = maxAllocation.getMemorySize(); + + // only emit max allocation if it differs from the parent's setting + if (maxVcores != parentMaxVcores || maxMemory != parentMaxMemory) { + capacitySchedulerConfig.set(PREFIX + queueName + + ".maximum-allocation-mb", String.valueOf(maxMemory)); + + capacitySchedulerConfig.set(PREFIX + queueName + + ".maximum-allocation-vcores", String.valueOf(maxVcores)); + } + } + } + + private void emitPreemptionDisabled(String queueName, FSQueue queue) { + if (preemptionEnabled && !queue.isPreemptable()) { + capacitySchedulerConfig.set(PREFIX + queueName + ".disable_preemption", + "true"); + } + } + + private void emitAutoCreateChildQueue(String queueName) { + if (autoCreateChildQueues) { + capacitySchedulerConfig.setBoolean(PREFIX + queueName + + ".auto-create-child-queue.enabled", true); + } + } + + private void emitSizeBasedWeight(String queueName) { + if (sizeBasedWeight) { + capacitySchedulerConfig.setBoolean(PREFIX + queueName + + ".ordering-policy.fair.enable-size-based-weight", true); + } + } + + private void emitOrderingPolicy(String queueName, FSQueue queue) { + String policy = queue.getPolicy().getName(); + + switch (policy) { + case FairSharePolicy.NAME: + capacitySchedulerConfig.set(PREFIX + queueName + + ".ordering-policy", FairSharePolicy.NAME); + fifoOrFairSharePolicyUsed = true; + break; + case FifoPolicy.NAME: + capacitySchedulerConfig.set(PREFIX + queueName + + ".ordering-policy", FifoPolicy.NAME); + fifoOrFairSharePolicyUsed = true; + break; + case DominantResourceFairnessPolicy.NAME: + // DRF is not supported on a queue level, + // it has to be global + drfPolicyUsedOnQueueLevel = true; + break; + default: + throw new IllegalArgumentException("Unexpected policy: " + policy); + } + } + + private void emitChildCapacity(FSQueue queue) { + List children = queue.getChildQueues(); + + int totalWeight = getTotalWeight(children); + Map capacities = getCapacities(totalWeight, children); + capacities + .entrySet() + .stream() + .forEach(entry -> { + capacitySchedulerConfig.set(PREFIX + entry.getKey() + ".capacity", + entry.getValue().toString()); + }); + } + + private void checkMaxChildCapacitySetting(FSQueue queue) { + if (queue.getMaxChildQueueResource() != null) { + Resource resource = queue.getMaxChildQueueResource().getResource(); + + if ((resource != null && Resources.unbounded().compareTo(resource) != 0) + || queue.getMaxChildQueueResource().getPercentages() != null) { + // Maximum child resource is defined + ruleHandler.handleMaxChildCapacity(); + } + } + } + + private Map getCapacities(int totalWeight, + List children) { + Map capacities = new HashMap<>(); + Map bdCapacities = new HashMap<>(); + final BigDecimal hundred = new BigDecimal(100).setScale(3); + + MutableBoolean needVerifySum = new MutableBoolean(true); + children + .stream() + .forEach(queue -> { + BigDecimal total = new BigDecimal(totalWeight); + BigDecimal weight = new BigDecimal(queue.getWeight()); + BigDecimal pct = weight + .setScale(5) + .divide(total, RoundingMode.HALF_UP) + .multiply(hundred) + .setScale(3); + + // defined? + if (Resources.none().compareTo(queue.getMinShare()) != 0) { + needVerifySum.setFalse(); + + /* TODO: Needs discussion. + * + * Probably it's not entirely correct this way! + * Eg. root.queue1 in FS translates to 33% + * capacity, but minResources is defined as 1vcore,8GB + * which is less than 33%. + * + * Therefore max(calculatedCapacity, minResource) is + * more sound. + */ + Resource minShare = queue.getMinShare(); + // TODO: in Phase-2, we have to deal with other resources as well + String capacity = String.format("[memory=%d,vcores=%d]", + minShare.getMemorySize(), minShare.getVirtualCores()); + capacities.put(queue.getName(), Capacity.newCapacity(capacity)); + } else { + capacities.put(queue.getName(), Capacity.newCapacity(pct)); + bdCapacities.put(queue.getName(), pct); + } + }); + + if (needVerifySum.isTrue()) { + BigDecimal totalPct = new BigDecimal(0); + for (Map.Entry entry : bdCapacities.entrySet()) { + totalPct = totalPct.add(entry.getValue()); + } + + // fix last value if total != 100.000 + if (!totalPct.equals(hundred)) { + if (children.size() > 1) { + BigDecimal tmp = new BigDecimal(0); + for (int i = 0; i < children.size() - 2; i++) { + tmp = tmp.add(bdCapacities.get(children.get(i).getQueueName())); + } + + String lastQueue = children.get(children.size() - 1).getName(); + BigDecimal corrected = hundred.subtract(tmp); + capacities.put(lastQueue, Capacity.newCapacity(corrected)); + } else if (children.size() == 1) { + // just a single element, probably this branch should + // not be reached + String queueName = children.get(0).getName(); + capacities.put(queueName, Capacity.newCapacity(hundred)); + } + } + } + + return capacities; + } + + private int getTotalWeight(List children) { + double sum = children + .stream() + .mapToDouble(c -> c.getWeight()) + .sum(); + return (int) sum; + } + + private String getQueueShortName(String queueName) { + int lastDot = queueName.lastIndexOf("."); + return queueName.substring(lastDot + 1); + } + + /* + * Represents a queue capacity in either percentage + * or in absolute resources + */ + private static class Capacity { + private BigDecimal percentage; + private String absoluteResource; + + public static Capacity newCapacity(BigDecimal pct) { + Capacity capacity = new Capacity(); + capacity.percentage = pct; + capacity.absoluteResource = null; + + return capacity; + } + + public static Capacity newCapacity(String absoluteResource) { + Capacity capacity = new Capacity(); + capacity.percentage = null; + capacity.absoluteResource = absoluteResource; + + return capacity; + } + + @Override + public String toString() { + if (percentage != null) { + return percentage.toString(); + } else { + return absoluteResource; + } + } + } + +} diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/converter/FSYarnSiteConverter.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/converter/FSYarnSiteConverter.java new file mode 100644 index 00000000000..a9c945716a8 --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/converter/FSYarnSiteConverter.java @@ -0,0 +1,159 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * http://www.apache.org/licenses/LICENSE-2.0 + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.converter; + +import static org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacitySchedulerConfiguration.PREFIX; + +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.yarn.conf.YarnConfiguration; +import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacityScheduler; +import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacitySchedulerConfiguration; +import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.FairSchedulerConfiguration; + +/** + * Converts a Fair Schedule site configuration to Capacity Scheduler + * site configuration. + * + */ +public class FSYarnSiteConverter { + private boolean preemptionEnabled; + private boolean autoCreateChildQueues; + private boolean sizeBasedWeight; + private boolean userAsDefaultQueue; + + @SuppressWarnings({"deprecation", "checkstyle:linelength"}) + public void convertSiteProperties(Configuration conf, + Configuration yarnSiteConfig) { + yarnSiteConfig.set(YarnConfiguration.RM_SCHEDULER, + CapacityScheduler.class.getCanonicalName()); + + // TODO: deprecated property, check if necessary + if (conf.getBoolean( + FairSchedulerConfiguration.CONTINUOUS_SCHEDULING_ENABLED, + FairSchedulerConfiguration.DEFAULT_CONTINUOUS_SCHEDULING_ENABLED)) { + yarnSiteConfig.setBoolean( + CapacitySchedulerConfiguration.SCHEDULE_ASYNCHRONOUSLY_ENABLE, true); + int interval = conf.getInt( + FairSchedulerConfiguration.CONTINUOUS_SCHEDULING_SLEEP_MS, + FairSchedulerConfiguration.DEFAULT_CONTINUOUS_SCHEDULING_SLEEP_MS); + yarnSiteConfig.setInt(PREFIX + + "schedule-asynchronously.scheduling-interval-ms", interval); + } + + String mbIncrementAllocation = + conf.get("yarn.resource-types.memory-mb.increment-allocation"); + if (mbIncrementAllocation != null) { + yarnSiteConfig.set("yarn.scheduler.minimum-allocation-mb", + mbIncrementAllocation); + } + + String vcoreIncrementAllocation = + conf.get("yarn.resource-types.vcores.increment-allocation"); + if (vcoreIncrementAllocation != null) { + yarnSiteConfig.set("yarn.scheduler.minimum-allocation-vcores", + vcoreIncrementAllocation); + } + + if (conf.getBoolean(FairSchedulerConfiguration.PREEMPTION, + FairSchedulerConfiguration.DEFAULT_PREEMPTION)) { + yarnSiteConfig.setBoolean( + YarnConfiguration.RM_SCHEDULER_ENABLE_MONITORS, true); + preemptionEnabled = true; + + int waitTimeBeforeKill = conf.getInt( + FairSchedulerConfiguration.WAIT_TIME_BEFORE_KILL, + FairSchedulerConfiguration.DEFAULT_WAIT_TIME_BEFORE_KILL); + yarnSiteConfig.setInt( + CapacitySchedulerConfiguration.PREEMPTION_WAIT_TIME_BEFORE_KILL, + waitTimeBeforeKill); + + long waitBeforeNextStarvationCheck = conf.getLong( + FairSchedulerConfiguration.WAIT_TIME_BEFORE_NEXT_STARVATION_CHECK_MS, + FairSchedulerConfiguration.DEFAULT_WAIT_TIME_BEFORE_NEXT_STARVATION_CHECK_MS); + yarnSiteConfig.setLong( + CapacitySchedulerConfiguration.PREEMPTION_MONITORING_INTERVAL, + waitBeforeNextStarvationCheck); + } + + if (conf.getBoolean(FairSchedulerConfiguration.ASSIGN_MULTIPLE, + FairSchedulerConfiguration.DEFAULT_ASSIGN_MULTIPLE)) { + yarnSiteConfig.setBoolean( + CapacitySchedulerConfiguration.ASSIGN_MULTIPLE_ENABLED, true); + } else { + yarnSiteConfig.setBoolean( + CapacitySchedulerConfiguration.ASSIGN_MULTIPLE_ENABLED, false); + } + + int maxAssign = conf.getInt(FairSchedulerConfiguration.MAX_ASSIGN, + FairSchedulerConfiguration.DEFAULT_MAX_ASSIGN); + if (maxAssign != -1) { + yarnSiteConfig.set( + CapacitySchedulerConfiguration.MAX_ASSIGN_PER_HEARTBEAT, + String.valueOf(maxAssign)); + } + + float localityThresholdNode = conf.getFloat( + FairSchedulerConfiguration.LOCALITY_THRESHOLD_NODE, + FairSchedulerConfiguration.DEFAULT_LOCALITY_THRESHOLD_NODE); + if (localityThresholdNode != + FairSchedulerConfiguration.DEFAULT_LOCALITY_THRESHOLD_NODE) { + yarnSiteConfig.setFloat(CapacitySchedulerConfiguration.NODE_LOCALITY_DELAY, + localityThresholdNode); + } + + float localityThresholdRack = conf.getFloat( + FairSchedulerConfiguration.LOCALITY_THRESHOLD_RACK, + FairSchedulerConfiguration.DEFAULT_LOCALITY_THRESHOLD_RACK); + if (localityThresholdRack != + FairSchedulerConfiguration.DEFAULT_LOCALITY_THRESHOLD_RACK) { + yarnSiteConfig.setFloat( + CapacitySchedulerConfiguration.RACK_LOCALITY_ADDITIONAL_DELAY, + localityThresholdRack); + } + + if (conf.getBoolean(FairSchedulerConfiguration.ALLOW_UNDECLARED_POOLS, + FairSchedulerConfiguration.DEFAULT_ALLOW_UNDECLARED_POOLS)) { + autoCreateChildQueues = true; + } + + if (conf.getBoolean(FairSchedulerConfiguration.SIZE_BASED_WEIGHT, + FairSchedulerConfiguration.DEFAULT_SIZE_BASED_WEIGHT)) { + sizeBasedWeight = true; + } + + if (conf.getBoolean(FairSchedulerConfiguration.USER_AS_DEFAULT_QUEUE, + FairSchedulerConfiguration.DEFAULT_USER_AS_DEFAULT_QUEUE)) { + userAsDefaultQueue = true; + } + } + + public boolean isPreemptionEnabled() { + return preemptionEnabled; + } + + public boolean isAutoCreateChildQueues() { + return autoCreateChildQueues; + } + + public boolean isSizeBasedWeight() { + return sizeBasedWeight; + } + + public boolean isUserAsDefaultQueue() { + return userAsDefaultQueue; + } +} diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/converter/QueuePlacementConverter.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/converter/QueuePlacementConverter.java new file mode 100644 index 00000000000..ea76ad357de --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/converter/QueuePlacementConverter.java @@ -0,0 +1,109 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * http://www.apache.org/licenses/LICENSE-2.0 + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.converter; + +import java.util.HashMap; +import java.util.Map; + +import org.apache.hadoop.yarn.server.resourcemanager.placement.DefaultPlacementRule; +import org.apache.hadoop.yarn.server.resourcemanager.placement.FSPlacementRule; +import org.apache.hadoop.yarn.server.resourcemanager.placement.PlacementManager; +import org.apache.hadoop.yarn.server.resourcemanager.placement.PlacementRule; +import org.apache.hadoop.yarn.server.resourcemanager.placement.PrimaryGroupPlacementRule; +import org.apache.hadoop.yarn.server.resourcemanager.placement.SecondaryGroupExistingPlacementRule; +import org.apache.hadoop.yarn.server.resourcemanager.placement.SpecifiedPlacementRule; +import org.apache.hadoop.yarn.server.resourcemanager.placement.UserPlacementRule; + +class QueuePlacementConverter { + + Map convertPlacementPolicy(PlacementManager placementManager, + FSConfigToCSConfigRuleHandler rulehandler, boolean userAsDefaultQueue) { + StringBuilder mapping = new StringBuilder(); + Map properties = new HashMap<>(); + + if (userAsDefaultQueue) { + mapping.append("u:%user:%user"); + } + + int ruleCount = 0; + for (PlacementRule rule : placementManager.getPlacementRules()) { + if (((FSPlacementRule)rule).getCreateFlag()) { + rulehandler.handleQueueAutoCreate(rule.getName()); + } + + ruleCount++; + if (rule instanceof UserPlacementRule) { + UserPlacementRule userRule = (UserPlacementRule) rule; + if (mapping.length() > 0) { + mapping.append(";"); + } + + // nested rule + if (userRule.getParentRule() != null) { + PlacementRule pr = userRule.getParentRule(); + if (pr instanceof PrimaryGroupPlacementRule) { + // TODO: wait for YARN-9841 + mapping.append("u:%user:%primary_group.%user"); + } else if (pr instanceof SecondaryGroupExistingPlacementRule) { + // TODO: wait for YARN-9865 + mapping.append("u:%user:%secondary_group.%user"); + } else if (pr instanceof DefaultPlacementRule) { + DefaultPlacementRule defaultRule = (DefaultPlacementRule) pr; + mapping.append("u:%user:") + .append(defaultRule.defaultQueueName) + .append(".%user"); + } else { + throw new UnsupportedOperationException("Unsupported nested rule: " + + pr.getClass().getCanonicalName()); + } + } else { + if (!userAsDefaultQueue) { + mapping.append("u:%user:%user"); + } + } + } else if (rule instanceof SpecifiedPlacementRule) { + if (ruleCount > 1) { + rulehandler.handleSpecifiedNotFirstRule(); + } + properties.put( + "yarn.scheduler.capacity.queue-mappings-override.enable", "false"); + } else if (rule instanceof PrimaryGroupPlacementRule) { + if (mapping.length() > 0) { + mapping.append(";"); + } + mapping.append("u:%user:%primary_group"); + } else if (rule instanceof DefaultPlacementRule) { + DefaultPlacementRule defaultRule = (DefaultPlacementRule) rule; + if (mapping.length() > 0) { + mapping.append(";"); + } + mapping.append("u:%user:").append(defaultRule.defaultQueueName); + } else if (rule instanceof SecondaryGroupExistingPlacementRule) { + // TODO: wait for YARN-9840 + mapping.append("u:%user:%secondary_group"); + } else { + throw new IllegalArgumentException("Unknown placement rule: " + rule); + } + } + + if (mapping.length() > 0) { + properties.put("yarn.scheduler.capacity.queue-mappings", + mapping.toString()); + } + + return properties; + } +} diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/converter/UnsupportedPropertyException.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/converter/UnsupportedPropertyException.java new file mode 100644 index 00000000000..8f89966618f --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/converter/UnsupportedPropertyException.java @@ -0,0 +1,29 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * http://www.apache.org/licenses/LICENSE-2.0 + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.converter; + +/** + * Thrown by the FS->CS converter if it encounters an + * unsupported property. + */ +public class UnsupportedPropertyException extends RuntimeException { + private static final long serialVersionUID = 5468104026818355871L; + + public UnsupportedPropertyException(String message) { + super(message); + } +} diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestResourceManager.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestResourceManager.java index f2073aeb55b..b1723cc0c68 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestResourceManager.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestResourceManager.java @@ -18,17 +18,8 @@ package org.apache.hadoop.yarn.server.resourcemanager; -import static org.junit.Assert.assertNotNull; -import static org.junit.Assert.fail; - -import java.io.IOException; -import java.util.Collection; -import java.util.concurrent.TimeoutException; - -import org.apache.hadoop.fs.CommonConfigurationKeysPublic; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.CommonConfigurationKeysPublic; import org.apache.hadoop.http.lib.StaticUserWebFilter; import org.apache.hadoop.net.NetworkTopology; import org.apache.hadoop.security.AuthenticationFilterInitializer; @@ -47,6 +38,9 @@ import org.apache.hadoop.yarn.server.resourcemanager.scheduler.event.AppAttemptRemovedSchedulerEvent; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.event.NodeAddedSchedulerEvent; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.event.NodeUpdateSchedulerEvent; +import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.converter.FSConfigConverterTestCommons; +import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.converter.FSConfigToCSConfigConverter; +import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.converter.FSConfigToCSConfigConverterParams; import org.apache.hadoop.yarn.server.security.http.RMAuthenticationFilterInitializer; import org.apache.hadoop.yarn.util.resource.Resources; import org.junit.After; @@ -55,11 +49,28 @@ import org.junit.Rule; import org.junit.Test; import org.junit.rules.ExpectedException; +import org.mockito.ArgumentCaptor; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import java.io.IOException; +import java.util.Collection; +import java.util.concurrent.TimeoutException; + +import static org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.converter.FSConfigConverterTestCommons.CONVERSION_RULES_FILE; +import static org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.converter.FSConfigConverterTestCommons.FS_ALLOC_FILE; +import static org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.converter.FSConfigConverterTestCommons.OUTPUT_DIR; +import static org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.converter.FSConfigConverterTestCommons.YARN_SITE_XML; +import static org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.converter.FSConfigConverterTestCommons.setupFSConfigConversionFiles; +import static org.assertj.core.api.Assertions.assertThat; +import static org.junit.Assert.assertNotNull; +import static org.junit.Assert.fail; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.verify; public class TestResourceManager { private static final Logger LOG = LoggerFactory.getLogger(TestResourceManager.class); - + private ResourceManager resourceManager = null; @Rule @@ -67,7 +78,7 @@ @Before public void setUp() throws Exception { - Configuration conf = new YarnConfiguration(); + YarnConfiguration conf = new YarnConfiguration(); UserGroupInformation.setConfiguration(conf); resourceManager = new ResourceManager(); resourceManager.init(conf); @@ -78,6 +89,7 @@ public void setUp() throws Exception { @After public void tearDown() throws Exception { resourceManager.stop(); + FSConfigConverterTestCommons.tearDown(); } private org.apache.hadoop.yarn.server.resourcemanager.NodeManager @@ -357,4 +369,70 @@ public void testUserProvidedUGIConf() throws Exception { } } + /** + * Example command:
+ * opt/hadoop/bin/yarn resourcemanager -convert-fs-configuration
+ * -o /tmp/output
+ * -y /opt/hadoop/etc/hadoop/yarn-site.xml
+ * -f /opt/hadoop/etc/hadoop/fair-scheduler.xml
+ * -r /home/systest/sample-rules-config.properties
+ */ + @Test + @SuppressWarnings("checkstyle:javadocstyle") + public void testResourceManagerConvertFSConfigurationDefaults() + throws Exception { + setupFSConfigConversionFiles(); + + ArgumentCaptor conversionParams = + ArgumentCaptor.forClass(FSConfigToCSConfigConverterParams.class); + + final String mainSwitch = "-convert-fs-configuration"; + FSConfigToCSConfigConverter mockConverter = + mock(FSConfigToCSConfigConverter.class); + + ResourceManager.initFSArgumentHandler(mockConverter); + ResourceManager.main(new String[] {mainSwitch, "-o", OUTPUT_DIR, + "-y", YARN_SITE_XML, "-f", FS_ALLOC_FILE, "-r", + CONVERSION_RULES_FILE}); + + // validate params + verify(mockConverter).convert(conversionParams.capture()); + FSConfigToCSConfigConverterParams params = conversionParams.getValue(); + LOG.info("FS config converter parameters: " + params); + + assertThat(params.getYarnSiteXmlConfig()).isEqualTo(YARN_SITE_XML); + assertThat(params.getFairSchedulerXmlConfig()).isEqualTo(FS_ALLOC_FILE); + assertThat(params.getConversionRulesConfig()) + .isEqualTo(CONVERSION_RULES_FILE); + assertThat(params.isConsole()).isEqualTo(false); + } + + @Test + public void testResourceManagerConvertFSConfigurationWithConsoleParam() + throws Exception { + setupFSConfigConversionFiles(); + + ArgumentCaptor conversionParams = + ArgumentCaptor.forClass(FSConfigToCSConfigConverterParams.class); + + final String mainSwitch = "-convert-fs-configuration"; + FSConfigToCSConfigConverter mockConverter = + mock(FSConfigToCSConfigConverter.class); + + ResourceManager.initFSArgumentHandler(mockConverter); + ResourceManager.main(new String[] {mainSwitch, "-o", OUTPUT_DIR, + "-p", "-y", YARN_SITE_XML, "-f", FS_ALLOC_FILE, "-r", + CONVERSION_RULES_FILE}); + + // validate params + verify(mockConverter).convert(conversionParams.capture()); + FSConfigToCSConfigConverterParams params = conversionParams.getValue(); + LOG.info("FS config converter parameters: " + params); + + assertThat(params.getYarnSiteXmlConfig()).isEqualTo(YARN_SITE_XML); + assertThat(params.getFairSchedulerXmlConfig()).isEqualTo(FS_ALLOC_FILE); + assertThat(params.getConversionRulesConfig()) + .isEqualTo(CONVERSION_RULES_FILE); + assertThat(params.isConsole()).isEqualTo(true); + } } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/converter/FSConfigConverterTestCommons.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/converter/FSConfigConverterTestCommons.java new file mode 100644 index 00000000000..93c5e0e96b7 --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/converter/FSConfigConverterTestCommons.java @@ -0,0 +1,164 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * http://www.apache.org/licenses/LICENSE-2.0 + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.converter; + +import org.apache.commons.lang3.StringUtils; +import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.FairSchedulerConfiguration; + +import java.io.ByteArrayOutputStream; +import java.io.File; +import java.io.FileWriter; +import java.io.IOException; +import java.io.PrintStream; +import java.io.PrintWriter; + +/** + * Helper methods for FS->CS converter testing. + * + */ +public class FSConfigConverterTestCommons { + private final static String TEST_DIR = + new File(System.getProperty("test.build.data", "/tmp")).getAbsolutePath(); + public final static String FS_ALLOC_FILE = + new File(TEST_DIR, "test-fair-scheduler.xml").getAbsolutePath(); + public final static String YARN_SITE_XML = + new File(TEST_DIR, "test-yarn-site.xml").getAbsolutePath(); + public final static String CONVERSION_RULES_FILE = + new File(TEST_DIR, "test-conversion-rules.properties").getAbsolutePath(); + public final static String OUTPUT_DIR = + new File(TEST_DIR, "conversion-output").getAbsolutePath(); + + private final ByteArrayOutputStream outContent = new ByteArrayOutputStream(); + private final ByteArrayOutputStream errContent = new ByteArrayOutputStream(); + + static { + new File(TEST_DIR, "conversion-output").mkdirs(); + } + + public static void tearDown() { + deleteTestFiles(); + restoreStreams(); + } + + public FSConfigConverterTestCommons() { + setUpStreams(); + } + + private void setUpStreams() { + System.setOut(new PrintStream(outContent)); + System.setErr(new PrintStream(errContent)); + } + + public static void restoreStreams() { + System.setOut(System.out); + System.setErr(System.err); + } + + public ByteArrayOutputStream getErrContent() { + return errContent; + } + + private static void deleteTestFiles() { + //Files may not be created so we are not strict here! + deleteFile(FS_ALLOC_FILE, false); + deleteFile(YARN_SITE_XML, false); + deleteFile(CONVERSION_RULES_FILE, false); + deleteFile(OUTPUT_DIR, false); + } + + private static void deleteFile(String f, boolean strict) { + boolean delete = new File(f).delete(); + if (strict && !delete) { + throw new RuntimeException("Can't delete test file: " + f); + } + } + + public static void setupFSConfigConversionFiles() throws IOException { + configureFairSchedulerXml(); + configureYarnSiteXmlWithFsAllocFileDefined(); + configureDummyConversionRulesFile(); + } + + @SuppressWarnings("checkstyle:linelength") + public static void configureFairSchedulerXml() throws IOException { + PrintWriter out = new PrintWriter(new FileWriter(FS_ALLOC_FILE)); + out.println(""); + out.println(""); + out.println("-1.0"); + out.println("fair"); + addQueue(out, ""); + out.println(""); + out.close(); + } + + @SuppressWarnings("checkstyle:linelength") + private static void addQueue(PrintWriter out, String additionalConfig) { + out.println(""); + out.println(" fair"); + out.println(" 1.0"); + out.println(" 100"); + out.println(" 120"); + out.println(" .5"); + + if (StringUtils.isNotEmpty(additionalConfig)) { + out.println(additionalConfig); + } + out.println(""); + } + + public static void configureEmptyFairSchedulerXml() throws IOException { + PrintWriter out = new PrintWriter(new FileWriter(FS_ALLOC_FILE)); + out.println(""); + out.println(""); + out.close(); + } + + public static void configureYarnSiteXmlWithFsAllocFileDefined() + throws IOException { + PrintWriter out = new PrintWriter(new FileWriter(YARN_SITE_XML)); + out.println(""); + out.println(""); + out.println("" + FairSchedulerConfiguration.ALLOCATION_FILE + ""); + out.println("" + FS_ALLOC_FILE + ""); + out.println(""); + out.close(); + } + + public static void configureEmptyYarnSiteXml() throws IOException { + PrintWriter out = new PrintWriter(new FileWriter(YARN_SITE_XML)); + out.println(""); + out.println(""); + out.close(); + } + + public static void configureDummyConversionRulesFile() throws IOException { + PrintWriter out = new PrintWriter(new FileWriter(CONVERSION_RULES_FILE)); + out.println("dummy_key=dummy_value"); + out.close(); + } + + public static void configureInvalidConversionRulesFile() throws IOException { + PrintWriter out = new PrintWriter(new FileWriter(CONVERSION_RULES_FILE)); + out.println("bla"); + out.close(); + } + + public static void configureEmptyConversionRulesFile() throws IOException { + PrintWriter out = new PrintWriter(new FileWriter(CONVERSION_RULES_FILE)); + out.close(); + } +} diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/converter/TestFSConfigToCSConfigArgumentHandler.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/converter/TestFSConfigToCSConfigArgumentHandler.java new file mode 100644 index 00000000000..d7da60d9af7 --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/converter/TestFSConfigToCSConfigArgumentHandler.java @@ -0,0 +1,360 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * http://www.apache.org/licenses/LICENSE-2.0 + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.converter; + +import com.google.common.collect.Lists; +import org.apache.commons.cli.MissingOptionException; +import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.ConversionException; +import org.junit.After; +import org.junit.Assert; +import org.junit.Before; +import org.junit.Rule; +import org.junit.Test; +import org.junit.rules.ExpectedException; +import org.mockito.ArgumentCaptor; +import org.mockito.ArgumentMatchers; +import org.mockito.Mockito; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.io.IOException; +import java.util.Arrays; +import java.util.List; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertTrue; + +/** + * Unit tests for FSConfigToCSConfigArgumentHandler. + * + */ +public class TestFSConfigToCSConfigArgumentHandler { + private static final Logger LOG = + LoggerFactory.getLogger(TestFSConfigToCSConfigArgumentHandler.class); + + @Rule + public ExpectedException expectedException = ExpectedException.none(); + + private FSConfigToCSConfigConverter mockConverter; + private FSConfigConverterTestCommons fsTestCommons; + + @Before + public void setUp() { + fsTestCommons = new FSConfigConverterTestCommons(); + } + + @After + public void tearDown() { + FSConfigConverterTestCommons.tearDown(); + } + + + private void setupFSConfigConversionFiles(boolean defineAllocationFile) + throws IOException { + FSConfigConverterTestCommons.configureFairSchedulerXml(); + + if (defineAllocationFile) { + FSConfigConverterTestCommons.configureYarnSiteXmlWithFsAllocFileDefined(); + } else { + FSConfigConverterTestCommons.configureEmptyYarnSiteXml(); + } + FSConfigConverterTestCommons.configureDummyConversionRulesFile(); + } + + + private FSConfigToCSConfigArgumentHandler createArgumentHandler() { + mockConverter = Mockito.mock(FSConfigToCSConfigConverter.class); + return new FSConfigToCSConfigArgumentHandler(mockConverter); + } + + private static String[] getDefaultArgumentsAsArray() { + List args = getDefaultArguments(); + return args.toArray(new String[0]); + } + + private static List getDefaultArguments() { + return Lists.newArrayList("-y", FSConfigConverterTestCommons.YARN_SITE_XML, "-o", FSConfigConverterTestCommons.OUTPUT_DIR); + } + + private String[] getArgumentsAsArrayWithDefaults(String... args) { + List result = getDefaultArguments(); + result.addAll(Arrays.asList(args)); + return result.toArray(new String[0]); + } + + private String[] getArgumentsAsArray(String... args) { + List result = Lists.newArrayList(); + result.addAll(Arrays.asList(args)); + return result.toArray(new String[0]); + } + + @Test + public void testMissingYarnSiteXmlArgument() throws Exception { + setupFSConfigConversionFiles(true); + + FSConfigToCSConfigArgumentHandler argumentHandler = + createArgumentHandler(); + + String[] args = new String[] {"-o", FSConfigConverterTestCommons.OUTPUT_DIR}; + + expectedException.expect(MissingOptionException.class); + expectedException.expectMessage("Missing required option: y"); + + argumentHandler.parseAndConvert(args); + } + + + @Test + public void testMissingFairSchedulerXmlArgument() throws Exception { + setupFSConfigConversionFiles(true); + FSConfigToCSConfigArgumentHandler argumentHandler = + createArgumentHandler(); + + argumentHandler.parseAndConvert(getDefaultArgumentsAsArray()); + } + + @Test + public void testMissingOutputDirArgument() throws Exception { + setupFSConfigConversionFiles(true); + + FSConfigToCSConfigArgumentHandler argumentHandler = + createArgumentHandler(); + + String[] args = new String[] {"-y", FSConfigConverterTestCommons.YARN_SITE_XML}; + + expectedException.expect(MissingOptionException.class); + expectedException.expectMessage("Missing required option: o"); + + argumentHandler.parseAndConvert(args); + } + + @Test + public void testMissingRulesConfiguration() throws Exception { + setupFSConfigConversionFiles(true); + FSConfigToCSConfigArgumentHandler argumentHandler = + createArgumentHandler(); + + argumentHandler.parseAndConvert(getDefaultArgumentsAsArray()); + } + + @Test + public void testInvalidRulesConfigFile() throws Exception { + FSConfigConverterTestCommons.configureYarnSiteXmlWithFsAllocFileDefined(); + FSConfigConverterTestCommons.configureFairSchedulerXml(); + FSConfigConverterTestCommons.configureInvalidConversionRulesFile(); + + FSConfigToCSConfigArgumentHandler argumentHandler = + createArgumentHandler(); + + String[] args = getArgumentsAsArrayWithDefaults(); + argumentHandler.parseAndConvert(args); + } + + @Test + public void testInvalidOutputDir() throws Exception { + FSConfigConverterTestCommons.configureYarnSiteXmlWithFsAllocFileDefined(); + FSConfigConverterTestCommons.configureFairSchedulerXml(); + FSConfigConverterTestCommons.configureDummyConversionRulesFile(); + + FSConfigToCSConfigArgumentHandler argumentHandler = + createArgumentHandler(); + + String[] args = getArgumentsAsArray("-y", + FSConfigConverterTestCommons.YARN_SITE_XML, "-o", FSConfigConverterTestCommons.YARN_SITE_XML); + + expectedException.expect(IllegalArgumentException.class); + expectedException.expectMessage("is not a directory"); + argumentHandler.parseAndConvert(args); + } + + @Test + public void testFairSchedulerXmlIsNotDefinedIfItsDefinedInYarnSiteXml() + throws Exception { + setupFSConfigConversionFiles(true); + FSConfigToCSConfigArgumentHandler argumentHandler = + createArgumentHandler(); + argumentHandler.parseAndConvert(getDefaultArgumentsAsArray()); + } + + @Test + public void testEmptyYarnSiteXmlSpecified() throws Exception { + FSConfigConverterTestCommons.configureFairSchedulerXml(); + FSConfigConverterTestCommons.configureEmptyYarnSiteXml(); + FSConfigConverterTestCommons.configureDummyConversionRulesFile(); + + FSConfigToCSConfigArgumentHandler argumentHandler = + createArgumentHandler(); + + String[] args = getArgumentsAsArrayWithDefaults("-f", FSConfigConverterTestCommons.FS_ALLOC_FILE); + argumentHandler.parseAndConvert(args); + } + + @Test + public void testEmptyFairSchedulerXmlSpecified() throws Exception { + FSConfigConverterTestCommons.configureEmptyFairSchedulerXml(); + FSConfigConverterTestCommons.configureEmptyYarnSiteXml(); + FSConfigConverterTestCommons.configureDummyConversionRulesFile(); + + FSConfigToCSConfigArgumentHandler argumentHandler = + createArgumentHandler(); + + String[] args = getArgumentsAsArrayWithDefaults("-f", FSConfigConverterTestCommons.FS_ALLOC_FILE); + argumentHandler.parseAndConvert(args); + } + + @Test + public void testEmptyRulesConfigurationSpecified() throws Exception { + FSConfigConverterTestCommons.configureEmptyFairSchedulerXml(); + FSConfigConverterTestCommons.configureEmptyYarnSiteXml(); + FSConfigConverterTestCommons.configureEmptyConversionRulesFile(); + + FSConfigToCSConfigArgumentHandler argumentHandler = + createArgumentHandler(); + + String[] args = getArgumentsAsArrayWithDefaults("-f", FSConfigConverterTestCommons.FS_ALLOC_FILE, + "-r", FSConfigConverterTestCommons.CONVERSION_RULES_FILE); + argumentHandler.parseAndConvert(args); + } + + @Test + public void testConvertFSConfigurationDefaults() throws Exception { + setupFSConfigConversionFiles(true); + + ArgumentCaptor conversionParams = + ArgumentCaptor.forClass(FSConfigToCSConfigConverterParams.class); + + FSConfigToCSConfigConverter mockConverter = + Mockito.mock(FSConfigToCSConfigConverter.class); + + FSConfigToCSConfigArgumentHandler argumentHandler = + new FSConfigToCSConfigArgumentHandler(mockConverter); + + String[] args = getArgumentsAsArrayWithDefaults("-f", FSConfigConverterTestCommons.FS_ALLOC_FILE, + "-r", FSConfigConverterTestCommons.CONVERSION_RULES_FILE); + argumentHandler.parseAndConvert(args); + + // validate params + Mockito.verify(mockConverter).convert(conversionParams.capture()); + FSConfigToCSConfigConverterParams params = conversionParams.getValue(); + LOG.info("FS config converter parameters: " + params); + + assertEquals("Yarn site config", + FSConfigConverterTestCommons.YARN_SITE_XML, params.getYarnSiteXmlConfig()); + assertEquals("FS xml", FSConfigConverterTestCommons.FS_ALLOC_FILE, params.getFairSchedulerXmlConfig()); + assertEquals("Conversion rules config", FSConfigConverterTestCommons.CONVERSION_RULES_FILE, + params.getConversionRulesConfig()); + assertFalse("Console mode", params.isConsole()); + } + + @Test + public void testConvertFSConfigurationWithConsoleParam() + throws Exception { + setupFSConfigConversionFiles(true); + + ArgumentCaptor conversionParams = + ArgumentCaptor.forClass(FSConfigToCSConfigConverterParams.class); + + FSConfigToCSConfigConverter mockConverter = + Mockito.mock(FSConfigToCSConfigConverter.class); + + FSConfigToCSConfigArgumentHandler argumentHandler = + new FSConfigToCSConfigArgumentHandler(mockConverter); + + String[] args = getArgumentsAsArrayWithDefaults("-f", FSConfigConverterTestCommons.FS_ALLOC_FILE, + "-r", FSConfigConverterTestCommons.CONVERSION_RULES_FILE, "-p"); + argumentHandler.parseAndConvert(args); + + // validate params + Mockito.verify(mockConverter).convert(conversionParams.capture()); + FSConfigToCSConfigConverterParams params = conversionParams.getValue(); + LOG.info("FS config converter parameters: " + params); + + assertEquals("Yarn site config", + FSConfigConverterTestCommons.YARN_SITE_XML, params.getYarnSiteXmlConfig()); + assertEquals("FS xml", FSConfigConverterTestCommons.FS_ALLOC_FILE, params.getFairSchedulerXmlConfig()); + assertEquals("Conversion rules config", FSConfigConverterTestCommons.CONVERSION_RULES_FILE, + params.getConversionRulesConfig()); + assertTrue("Console mode", params.isConsole()); + } + + @Test + public void testConvertFSConfigurationClusterResource() + throws Exception { + setupFSConfigConversionFiles(true); + + ArgumentCaptor conversionParams = + ArgumentCaptor.forClass(FSConfigToCSConfigConverterParams.class); + + FSConfigToCSConfigConverter mockConverter = + Mockito.mock(FSConfigToCSConfigConverter.class); + + FSConfigToCSConfigArgumentHandler argumentHandler = + new FSConfigToCSConfigArgumentHandler(mockConverter); + + String[] args = getArgumentsAsArrayWithDefaults("-f", FSConfigConverterTestCommons.FS_ALLOC_FILE, + "-r", FSConfigConverterTestCommons.CONVERSION_RULES_FILE, "-p", "-c", "vcores=20, memory-mb=240"); + argumentHandler.parseAndConvert(args); + + // validate params + Mockito.verify(mockConverter).convert(conversionParams.capture()); + FSConfigToCSConfigConverterParams params = conversionParams.getValue(); + LOG.info("FS config converter parameters: " + params); + + assertEquals("Yarn site config", + FSConfigConverterTestCommons.YARN_SITE_XML, params.getYarnSiteXmlConfig()); + assertEquals("FS xml", FSConfigConverterTestCommons.FS_ALLOC_FILE, params.getFairSchedulerXmlConfig()); + assertEquals("Conversion rules config", FSConfigConverterTestCommons.CONVERSION_RULES_FILE, + params.getConversionRulesConfig()); + assertEquals("Cluster resource", "vcores=20, memory-mb=240", + params.getClusterResource()); + assertTrue("Console mode", params.isConsole()); + } + + @Test + public void testConvertFSConfigurationErrorHandling() throws Exception { + setupFSConfigConversionFiles(true); + + String[] args = getArgumentsAsArrayWithDefaults("-f", FSConfigConverterTestCommons.FS_ALLOC_FILE, + "-r", FSConfigConverterTestCommons.CONVERSION_RULES_FILE, "-p"); + FSConfigToCSConfigArgumentHandler argumentHandler = + createArgumentHandler(); + + Mockito.doThrow(UnsupportedPropertyException.class) + .when(mockConverter) + .convert(ArgumentMatchers.any(FSConfigToCSConfigConverterParams.class)); + argumentHandler.parseAndConvert(args); + assertTrue("Error content missing", fsTestCommons.getErrContent() + .toString().contains("Fatal error during conversion!")); + } + + @Test + public void testConvertFSConfigurationErrorHandling2() throws Exception { + setupFSConfigConversionFiles(true); + + String[] args = getArgumentsAsArrayWithDefaults("-f", FSConfigConverterTestCommons.FS_ALLOC_FILE, + "-r", FSConfigConverterTestCommons.CONVERSION_RULES_FILE, "-p"); + FSConfigToCSConfigArgumentHandler argumentHandler = + createArgumentHandler(); + + Mockito.doThrow(ConversionException.class).when(mockConverter) + .convert(ArgumentMatchers.any(FSConfigToCSConfigConverterParams.class)); + argumentHandler.parseAndConvert(args); + assertTrue("Error content missing", fsTestCommons.getErrContent() + .toString().contains("Error occurred during property conversion")); + } +} \ No newline at end of file diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/converter/TestFSConfigToCSConfigConverter.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/converter/TestFSConfigToCSConfigConverter.java new file mode 100644 index 00000000000..2939e39e6cd --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/converter/TestFSConfigToCSConfigConverter.java @@ -0,0 +1,448 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * http://www.apache.org/licenses/LICENSE-2.0 + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.converter; + +import static org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacitySchedulerConfiguration.PREFIX; +import static org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.converter.FSConfigToCSConfigRuleHandler.DYNAMIC_MAX_ASSIGN; +import static org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.converter.FSConfigToCSConfigRuleHandler.MAX_CAPACITY_PERCENTAGE; +import static org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.converter.FSConfigToCSConfigRuleHandler.MAX_CHILD_CAPACITY; +import static org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.converter.FSConfigToCSConfigRuleHandler.QUEUE_AUTO_CREATE; +import static org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.converter.FSConfigToCSConfigRuleHandler.RESERVATION_SYSTEM; +import static org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.converter.FSConfigToCSConfigRuleHandler.SPECIFIED_NOT_FIRST; +import static org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.converter.FSConfigToCSConfigRuleHandler.USER_MAX_APPS_DEFAULT; +import static org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.converter.FSConfigToCSConfigRuleHandler.USER_MAX_RUNNING_APPS; +import static org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.converter.FSConfigToCSConfigRuleHandler.RuleAction.ABORT; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertTrue; +import static org.junit.Assert.fail; + +import java.io.ByteArrayInputStream; +import java.io.ByteArrayOutputStream; +import java.io.File; +import java.io.IOException; +import java.util.Map; + +import org.apache.commons.io.FileUtils; +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.yarn.api.records.Resource; +import org.apache.hadoop.yarn.conf.YarnConfiguration; +import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacitySchedulerConfiguration; +import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.ConversionException; +import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.FairSchedulerConfiguration; +import org.junit.Assert; +import org.junit.Before; +import org.junit.Rule; +import org.junit.Test; +import org.junit.rules.ExpectedException; +import org.junit.runner.RunWith; +import org.mockito.Mock; +import org.mockito.Mockito; +import org.mockito.junit.MockitoJUnitRunner; + + +/** + * Unit tests for FSConfigToCSConfigConverter. + * + */ +@RunWith(MockitoJUnitRunner.class) +public class TestFSConfigToCSConfigConverter { + private static final Resource CLUSTER_RESOURCE = + Resource.newInstance(16384, 16); + private static final String FILE_PREFIX = "file:"; + private static final String FAIR_SCHEDULER_XML = + prepareFileName("fair-scheduler-conversion.xml"); + + @Mock + private FSConfigToCSConfigRuleHandler ruleHandler; + + private FSConfigToCSConfigConverter converter; + private Configuration config; + + private ByteArrayOutputStream csConfigOut; + + @Rule + public ExpectedException expectedException = ExpectedException.none(); + + private static String prepareFileName(String f) { + return FILE_PREFIX + new File("src/test/resources/" + f).getAbsolutePath(); + } + + private static final String FAIR_SCHEDULER_XML_INVALID = + prepareFileName("fair-scheduler-invalid.xml"); + private static final String YARN_SITE_XML = + prepareFileName("yarn-site-with-allocation-file-ref.xml"); + private static final String YARN_SITE_XML_NO_REF_TO_FS_XML = + prepareFileName("yarn-site.xml"); + private static final String YARN_SITE_XML_INVALID = + prepareFileName("yarn-site-with-invalid-allocation-file-ref.xml"); + private static final String CONVERSION_RULES_FILE = + new File("src/test/resources/conversion-rules.properties") + .getAbsolutePath(); + + @Before + public void setup() { + config = new Configuration(false); + config.set(FairSchedulerConfiguration.ALLOCATION_FILE, FAIR_SCHEDULER_XML); + config.setBoolean(FairSchedulerConfiguration.MIGRATION_MODE, true); + createConverter(); + } + + private void createConverter() { + converter = new FSConfigToCSConfigConverter(ruleHandler); + ByteArrayOutputStream yarnSiteOut = new ByteArrayOutputStream(); + csConfigOut = new ByteArrayOutputStream(); + + converter.setCapacitySchedulerConfigOutputStream(csConfigOut); + converter.setYarnSiteOutputStream(yarnSiteOut); + } + + private FSConfigToCSConfigConverterParams.Builder + createDefaultParamsBuilder() { + return FSConfigToCSConfigConverterParams.Builder.create() + .withYarnSiteXmlConfig(YARN_SITE_XML) + .withOutputDirectory(FSConfigConverterTestCommons.OUTPUT_DIR); + } + + private FSConfigToCSConfigConverterParams.Builder + createParamsBuilder(String yarnSiteConfig) { + return FSConfigToCSConfigConverterParams.Builder.create() + .withYarnSiteXmlConfig(yarnSiteConfig) + .withOutputDirectory(FSConfigConverterTestCommons.OUTPUT_DIR); + } + + @Test + public void testDefaultMaxApplications() throws Exception { + converter.convert(config, CLUSTER_RESOURCE); + + Configuration conf = getConvertedCSConfig(); + int maxApps = + conf.getInt( + CapacitySchedulerConfiguration.MAXIMUM_SYSTEM_APPLICATIONS, -1); + + assertEquals("Default max apps", 15, maxApps); + } + + @Test + public void testDefaultMaxAMShare() throws Exception { + converter.convert(config, CLUSTER_RESOURCE); + + Configuration conf = getConvertedCSConfig(); + String maxAmShare = + conf.get(CapacitySchedulerConfiguration. + MAXIMUM_APPLICATION_MASTERS_RESOURCE_PERCENT); + + assertEquals("Default max AM share", "0.16", maxAmShare); + } + + @Test + public void testConvertACLs() throws Exception { + converter.convert(config, CLUSTER_RESOURCE); + + Configuration conf = getConvertedCSConfig(); + + // root + assertEquals("root submit ACL", "alice,bob,joe,john hadoop_users", + conf.get(PREFIX + "root.acl_submit_applications")); + assertEquals("root admin ACL", "alice,bob,joe,john hadoop_users", + conf.get(PREFIX + "root.acl_administer_queue")); + + // root.admins.bob + assertEquals("root.admins.bob submit ACL", "bob ", + conf.get(PREFIX + "root.admins.bob.acl_submit_applications")); + assertEquals("root.admins.bob admin ACL", "bob ", + conf.get(PREFIX + "root.admins.bob.acl_administer_queue")); + + // root.admins.alice + assertEquals("root.admins.alice submit ACL", "alice ", + conf.get(PREFIX + "root.admins.alice.acl_submit_applications")); + assertEquals("root.admins.alice admin ACL", "alice ", + conf.get(PREFIX + "root.admins.alice.acl_administer_queue")); + + // root.users.john + assertEquals("root.users.john submit ACL", "john ", + conf.get(PREFIX + "root.users.john.acl_submit_applications")); + assertEquals("root.users.john admin ACL", "john ", + conf.get(PREFIX + "root.users.john.acl_administer_queue")); + + // root.users.joe + assertEquals("root.users.joe submit ACL", "joe ", + conf.get(PREFIX + "root.users.joe.acl_submit_applications")); + assertEquals("root.users.joe admin ACL", "joe ", + conf.get(PREFIX + "root.users.joe.acl_administer_queue")); + } + + @Test + public void testDefaultMaxRunningApps() throws Exception { + converter.convert(config, CLUSTER_RESOURCE); + + Configuration conf = getConvertedCSConfig(); + + // default setting + assertEquals("Default max apps", 15, + conf.getInt(PREFIX + "maximum-applications", -1)); + } + + @Test + public void testMixedQueueOrderingPolicy() throws Exception { + expectedException.expect(ConversionException.class); + expectedException.expectMessage( + "DRF ordering policy cannot be used together with fifo/fair"); + String absolutePath = + new File("src/test/resources/fair-scheduler-orderingpolicy-mixed.xml") + .getAbsolutePath(); + config.set(FairSchedulerConfiguration.ALLOCATION_FILE, + FILE_PREFIX + absolutePath); + + converter.convert(config, CLUSTER_RESOURCE); + } + + @Test + public void testQueueMaxChildCapacityNotSupported() throws Exception { + expectedException.expect(UnsupportedPropertyException.class); + expectedException.expectMessage("test"); + + Mockito.doThrow(new UnsupportedPropertyException("test")) + .when(ruleHandler).handleMaxChildCapacity(); + + converter.convert(config, CLUSTER_RESOURCE); + } + + @Test + public void testReservationSystemNotSupported() throws Exception { + expectedException.expect(UnsupportedPropertyException.class); + expectedException.expectMessage("maxCapacity"); + + Mockito.doThrow(new UnsupportedPropertyException("maxCapacity")) + .when(ruleHandler).handleMaxChildCapacity(); + config.setBoolean(YarnConfiguration.RM_RESERVATION_SYSTEM_ENABLE, true); + + converter.convert(config, CLUSTER_RESOURCE); + } + + @Test + public void testUserMaxAppsNotSupported() throws Exception { + expectedException.expect(UnsupportedPropertyException.class); + expectedException.expectMessage("userMaxApps"); + + Mockito.doThrow(new UnsupportedPropertyException("userMaxApps")) + .when(ruleHandler).handleUserMaxApps(); + + converter.convert(config, CLUSTER_RESOURCE); + } + + @Test + public void testUserMaxAppsDefaultNotSupported() throws Exception { + expectedException.expect(UnsupportedPropertyException.class); + expectedException.expectMessage("userMaxAppsDefault"); + + Mockito.doThrow(new UnsupportedPropertyException("userMaxAppsDefault")) + .when(ruleHandler).handleUserMaxAppsDefault(); + + converter.convert(config, CLUSTER_RESOURCE); + } + + @Test + public void testConvertFSConfigurationClusterResource() throws Exception { + FSConfigToCSConfigConverterParams params = createDefaultParamsBuilder() + .withClusterResource("vcores=20, memory-mb=240") + .build(); + converter.convert(params); + assertEquals("Resource", Resource.newInstance(240, 20), + converter.getClusterResource()); + } + + @Test + public void testConvertFSConfigPctModeUsedAndClusterResourceDefined() + throws Exception { + FSConfigToCSConfigConverterParams params = createDefaultParamsBuilder() + .withClusterResource("vcores=20, memory-mb=240") + .build(); + converter.convert(params); + assertEquals("Resource", Resource.newInstance(240, 20), + converter.getClusterResource()); + } + + @Test + public void testConvertFSConfigPctModeUsedAndClusterResourceNotDefined() + throws Exception { + FSConfigToCSConfigConverterParams params = createDefaultParamsBuilder() + .build(); + + expectedException.expect(ConversionException.class); + expectedException.expectMessage("cluster resource parameter" + + " is not defined via CLI"); + + converter.convert(params); + } + + @Test + public void testConvertFSConfigurationClusterResourceInvalid() + throws Exception { + FSConfigToCSConfigConverterParams params = createDefaultParamsBuilder() + .withClusterResource("vcores=20, memory-mb=240G") + .build(); + + expectedException.expect(ConversionException.class); + expectedException.expectMessage("Error while parsing resource"); + + converter.convert(params); + } + + @Test + public void testConvertFSConfigurationClusterResourceInvalid2() + throws Exception { + FSConfigToCSConfigConverterParams params = createDefaultParamsBuilder() + .withClusterResource("vcores=20, memmmm=240") + .build(); + + expectedException.expect(ConversionException.class); + expectedException.expectMessage("Error while parsing resource"); + + converter.convert(params); + } + + @Test + public void testConvertFSConfigurationRulesFile() throws Exception { + ruleHandler = new FSConfigToCSConfigRuleHandler(); + createConverter(); + + FSConfigToCSConfigConverterParams params = + createDefaultParamsBuilder() + .withConversionRulesConfig(CONVERSION_RULES_FILE) + .withClusterResource("vcores=20, memory-mb=2400") + .build(); + + try { + converter.convert(params); + fail("Should have thrown UnsupportedPropertyException!"); + } catch (UnsupportedPropertyException e) { + //need to catch exception so we can check the rules + } + + ruleHandler = converter.getRuleHandler(); + Map actions = + ruleHandler.getActions(); + + assertEquals("maxCapacityPercentage", + ABORT, actions.get(MAX_CAPACITY_PERCENTAGE)); + assertEquals("maxChildCapacity", + ABORT, actions.get(MAX_CHILD_CAPACITY)); + assertEquals("userMaxRunningApps", + ABORT, actions.get(USER_MAX_RUNNING_APPS)); + assertEquals("userMaxAppsDefault", + ABORT, actions.get(USER_MAX_APPS_DEFAULT)); + assertEquals("dynamicMaxAssign", + ABORT, actions.get(DYNAMIC_MAX_ASSIGN)); + assertEquals("specifiedNotFirstRule", + ABORT, actions.get(SPECIFIED_NOT_FIRST)); + assertEquals("reservationSystem", + ABORT, actions.get(RESERVATION_SYSTEM)); + assertEquals("queueAutoCreate", + ABORT, actions.get(QUEUE_AUTO_CREATE)); + } + + @Test + public void testConvertFSConfigurationUndefinedYarnSiteConfig() + throws Exception { + FSConfigToCSConfigConverterParams params = + FSConfigToCSConfigConverterParams.Builder.create() + .withYarnSiteXmlConfig(null) + .withOutputDirectory(FSConfigConverterTestCommons.OUTPUT_DIR) + .build(); + + expectedException.expect(IllegalArgumentException.class); + expectedException.expectMessage( + "yarn-site.xml configuration is not defined"); + + converter.convert(params); + } + + @Test + public void testConvertCheckOutputDir() throws Exception { + FSConfigToCSConfigConverterParams params = createDefaultParamsBuilder() + .withClusterResource("vcores=20, memory-mb=240") + .build(); + + converter.convert(params); + + Configuration conf = getConvertedCSConfig(FSConfigConverterTestCommons.OUTPUT_DIR); + + File capacityFile = new File(FSConfigConverterTestCommons.OUTPUT_DIR, "capacity-scheduler.xml"); + assertTrue("Capacity file exists", capacityFile.exists()); + assertTrue("Capacity file length > 0", capacityFile.length() > 0); + assertTrue("No. of configuration elements > 0", conf.size() > 0); + + File yarnSiteFile = new File(FSConfigConverterTestCommons.OUTPUT_DIR, "yarn-site.xml"); + assertTrue("Yarn site exists", yarnSiteFile.exists()); + assertTrue("Yarn site length > 0", yarnSiteFile.length() > 0); + } + + @Test + public void testFairSchedulerXmlIsNotDefinedNeitherDirectlyNorInYarnSiteXml() + throws Exception { + FSConfigToCSConfigConverterParams params = + createParamsBuilder(YARN_SITE_XML_NO_REF_TO_FS_XML) + .withClusterResource("vcores=20, memory-mb=240") + .build(); + + expectedException.expect(IllegalArgumentException.class); + expectedException.expectMessage("fair-scheduler.xml is not defined"); + converter.convert(params); + } + + @Test + public void testInvalidFairSchedulerXml() throws Exception { + FSConfigToCSConfigConverterParams params = createDefaultParamsBuilder() + .withClusterResource("vcores=20, memory-mb=240") + .withFairSchedulerXmlConfig(FAIR_SCHEDULER_XML_INVALID) + .build(); + + expectedException.expect(RuntimeException.class); + converter.convert(params); + } + + @Test + public void testInvalidYarnSiteXml() throws Exception { + FSConfigToCSConfigConverterParams params = + createParamsBuilder(YARN_SITE_XML_INVALID) + .withClusterResource("vcores=20, memory-mb=240") + .build(); + + expectedException.expect(RuntimeException.class); + converter.convert(params); + } + + private Configuration getConvertedCSConfig() { + ByteArrayInputStream input = + new ByteArrayInputStream(csConfigOut.toByteArray()); + assertTrue("CS config output has length of 0!", csConfigOut.toByteArray().length > 0); + Configuration conf = new Configuration(false); + conf.addResource(input); + + return conf; + } + + private Configuration getConvertedCSConfig(String dir) throws IOException { + File capacityFile = new File(dir, "capacity-scheduler.xml"); + ByteArrayInputStream input = + new ByteArrayInputStream(FileUtils.readFileToByteArray(capacityFile)); + Configuration conf = new Configuration(false); + conf.addResource(input); + + return conf; + } +} diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/converter/TestFSConfigToCSConfigRuleHandler.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/converter/TestFSConfigToCSConfigRuleHandler.java new file mode 100644 index 00000000000..f3d73b9cbc0 --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/converter/TestFSConfigToCSConfigRuleHandler.java @@ -0,0 +1,149 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * http://www.apache.org/licenses/LICENSE-2.0 + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.converter; + +import static org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.converter.FSConfigToCSConfigRuleHandler.DYNAMIC_MAX_ASSIGN; +import static org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.converter.FSConfigToCSConfigRuleHandler.MAX_CAPACITY_PERCENTAGE; +import static org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.converter.FSConfigToCSConfigRuleHandler.MAX_CHILD_CAPACITY; +import static org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.converter.FSConfigToCSConfigRuleHandler.MAX_CHILD_QUEUE_LIMIT; +import static org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.converter.FSConfigToCSConfigRuleHandler.QUEUE_AUTO_CREATE; +import static org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.converter.FSConfigToCSConfigRuleHandler.RESERVATION_SYSTEM; +import static org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.converter.FSConfigToCSConfigRuleHandler.SPECIFIED_NOT_FIRST; +import static org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.converter.FSConfigToCSConfigRuleHandler.USER_MAX_APPS_DEFAULT; +import static org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.converter.FSConfigToCSConfigRuleHandler.USER_MAX_RUNNING_APPS; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertTrue; + +import java.io.IOException; +import java.util.Properties; + +import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.ConversionException; +import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.converter.FSConfigToCSConfigRuleHandler; +import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.converter.UnsupportedPropertyException; +import org.junit.Assert; +import org.junit.Test; + +/** + * Unit tests for FSConfigToCSConfigRuleHandler. + * + */ +public class TestFSConfigToCSConfigRuleHandler { + private static final String ABORT = "abort"; + private static final String WARNING = "warning"; + + private FSConfigToCSConfigRuleHandler ruleHandler; + + @Test + public void testInitPropertyActionsToWarning() throws IOException { + ruleHandler = new FSConfigToCSConfigRuleHandler(new Properties()); + + ruleHandler.handleChildQueueCount("test", 1); + ruleHandler.handleDynamicMaxAssign(); + ruleHandler.handleMaxCapacityPercentage("test"); + ruleHandler.handleMaxChildCapacity(); + ruleHandler.handleQueueAutoCreate("test"); + ruleHandler.handleReservationSystem(); + ruleHandler.handleSpecifiedNotFirstRule(); + ruleHandler.handleUserMaxApps(); + ruleHandler.handleUserMaxAppsDefault(); + } + + @Test + public void testAllRulesWarning() throws IOException { + Properties rules = new Properties(); + rules.put(DYNAMIC_MAX_ASSIGN, WARNING); + rules.put(MAX_CAPACITY_PERCENTAGE, WARNING); + rules.put(MAX_CHILD_CAPACITY, WARNING); + rules.put(QUEUE_AUTO_CREATE, WARNING); + rules.put(RESERVATION_SYSTEM, WARNING); + rules.put(SPECIFIED_NOT_FIRST, WARNING); + rules.put(USER_MAX_APPS_DEFAULT, WARNING); + rules.put(USER_MAX_RUNNING_APPS, WARNING); + + ruleHandler = new FSConfigToCSConfigRuleHandler(rules); + + ruleHandler.handleDynamicMaxAssign(); + ruleHandler.handleMaxCapacityPercentage("test"); + ruleHandler.handleMaxChildCapacity(); + ruleHandler.handleQueueAutoCreate("test"); + ruleHandler.handleReservationSystem(); + ruleHandler.handleSpecifiedNotFirstRule(); + ruleHandler.handleUserMaxApps(); + ruleHandler.handleUserMaxAppsDefault(); + } + + @Test + public void testAllRulesAbort() throws IOException { + Properties rules = new Properties(); + rules.put(DYNAMIC_MAX_ASSIGN, ABORT); + rules.put(MAX_CAPACITY_PERCENTAGE, ABORT); + rules.put(MAX_CHILD_CAPACITY, ABORT); + rules.put(QUEUE_AUTO_CREATE, ABORT); + rules.put(RESERVATION_SYSTEM, ABORT); + rules.put(SPECIFIED_NOT_FIRST, ABORT); + rules.put(USER_MAX_APPS_DEFAULT, ABORT); + rules.put(USER_MAX_RUNNING_APPS, ABORT); + rules.put(MAX_CHILD_QUEUE_LIMIT, "1"); + + ruleHandler = new FSConfigToCSConfigRuleHandler(rules); + + expectAbort(() -> ruleHandler.handleChildQueueCount("test", 2), + ConversionException.class); + expectAbort(() -> ruleHandler.handleDynamicMaxAssign()); + expectAbort(() -> ruleHandler.handleMaxCapacityPercentage("test")); + expectAbort(() -> ruleHandler.handleMaxChildCapacity()); + expectAbort(() -> ruleHandler.handleQueueAutoCreate("test")); + expectAbort(() -> ruleHandler.handleReservationSystem()); + expectAbort(() -> ruleHandler.handleSpecifiedNotFirstRule()); + expectAbort(() -> ruleHandler.handleUserMaxApps()); + expectAbort(() -> ruleHandler.handleUserMaxAppsDefault()); + } + + @Test(expected = IllegalArgumentException.class) + public void testMaxChildQueueCountNotInteger() throws IOException { + Properties rules = new Properties(); + rules.put(MAX_CHILD_QUEUE_LIMIT, "abc"); + + ruleHandler = new FSConfigToCSConfigRuleHandler(rules); + + ruleHandler.handleChildQueueCount("test", 1); + } + + private void expectAbort(VoidCall call) { + expectAbort(call, UnsupportedPropertyException.class); + } + + private void expectAbort(VoidCall call, Class exceptionClass) { + boolean exceptionThrown = false; + Throwable thrown = null; + + try { + call.apply(); + } catch (Throwable t) { + thrown = t; + exceptionThrown = true; + } + + assertTrue("Exception was not thrown", exceptionThrown); + assertEquals("Unexpected exception", exceptionClass, thrown.getClass()); + } + + @FunctionalInterface + private interface VoidCall { + void apply(); + } +} diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/converter/TestFSQueueConverter.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/converter/TestFSQueueConverter.java new file mode 100644 index 00000000000..4ee76ece487 --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/converter/TestFSQueueConverter.java @@ -0,0 +1,432 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * http://www.apache.org/licenses/LICENSE-2.0 + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.converter; + +import static org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacitySchedulerConfiguration.PREFIX; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertNull; +import static org.junit.Assert.assertTrue; + +import java.io.File; +import java.io.IOException; +import java.util.Set; + +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.yarn.api.records.Resource; +import org.apache.hadoop.yarn.conf.YarnConfiguration; +import org.apache.hadoop.yarn.server.resourcemanager.RMContext; +import org.apache.hadoop.yarn.server.resourcemanager.RMContextImpl; +import org.apache.hadoop.yarn.server.resourcemanager.placement.PlacementManager; +import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.ConversionException; +import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.FSQueue; +import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.FairScheduler; +import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.FairSchedulerConfiguration; +import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.converter.FSConfigToCSConfigRuleHandler; +import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.converter.FSQueueConverter; +import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.converter.UnsupportedPropertyException; +import org.junit.After; +import org.junit.Assert; +import org.junit.Before; +import org.junit.Rule; +import org.junit.Test; +import org.junit.rules.ExpectedException; +import org.junit.runner.RunWith; +import org.mockito.Mock; +import org.mockito.Mockito; +import org.mockito.junit.MockitoJUnitRunner; + +import com.google.common.collect.Sets; + + +/** + * Unit tests for FSQueueConverter. + * + */ +@RunWith(MockitoJUnitRunner.class) +public class TestFSQueueConverter { + private static final Resource CLUSTER_RESOURCE = + Resource.newInstance(16384, 16); + private final static Set ALL_QUEUES = + Sets.newHashSet("root", + "root.default", + "root.admins", + "root.users", + "root.admins.alice", + "root.admins.bob", + "root.users.joe", + "root.users.john"); + + private static final String FILE_PREFIX = "file:"; + private static final String FAIR_SCHEDULER_XML = + prepareFileName("fair-scheduler-conversion.xml"); + + private static String prepareFileName(String f) { + return FILE_PREFIX + new File("src/test/resources/" + f).getAbsolutePath(); + } + + private FSQueueConverter converter; + private Configuration config; + private Configuration csConfig; + private FairScheduler fs; + private FSQueue rootQueue; + + @Mock + private FSConfigToCSConfigRuleHandler ruleHandler; + + @Rule + public ExpectedException expectedException = ExpectedException.none(); + + @Before + public void setup() { + config = new Configuration(false); + config.set(FairSchedulerConfiguration.ALLOCATION_FILE, FAIR_SCHEDULER_XML); + config.setBoolean(FairSchedulerConfiguration.MIGRATION_MODE, true); + csConfig = new Configuration(false); + + fs = createFairScheduler(); + + createConverter(); + rootQueue = fs.getQueueManager().getRootQueue(); + } + + @After + public void tearDown() throws IOException { + if (fs != null) { + fs.close(); + } + } + + private FairScheduler createFairScheduler() { + RMContext ctx = new RMContextImpl(); + PlacementManager placementManager = new PlacementManager(); + ctx.setQueuePlacementManager(placementManager); + + FairScheduler fairScheduler = new FairScheduler(); + fairScheduler.setRMContext(ctx); + fairScheduler.init(config); + + return fairScheduler; + } + + private void createConverter() { + converter = new FSQueueConverter(ruleHandler, + csConfig, + false, + false, + false, + CLUSTER_RESOURCE, + 0.16f, + 15); + } + + @Test + public void testConvertQueueHierarchy() { + converter.convertQueue(rootQueue); + + // root children + assertEquals("root children", "default,admins,users", + csConfig.get(PREFIX + "root.queues")); + + // root.admins children + assertEquals("root.admins children", "bob,alice", + csConfig.get(PREFIX + "root.admins.queues")); + + // root.default children - none + assertNull("root.default children", csConfig.get(PREFIX + "root.default" + + ".queues")); + + // root.users children + assertEquals("root.users children", "john,joe", + csConfig.get(PREFIX + "root.users.queues")); + + Set leafs = Sets.difference(ALL_QUEUES, + Sets.newHashSet("root", + "root.default", + "root.admins", + "root.users")); + + assertNoValueForQueues(leafs, ".queues", csConfig); + } + + @Test + public void testConvertQueueHierarchyWithSameLeafQueues() throws Exception { + expectedException.expect(ConversionException.class); + expectedException.expectMessage("Leaf queues must be unique"); + + String absolutePath = + new File("src/test/resources/fair-scheduler-sameleafqueue.xml") + .getAbsolutePath(); + config.set(FairSchedulerConfiguration.ALLOCATION_FILE, + FILE_PREFIX + absolutePath); + fs.close(); + fs = createFairScheduler(); + rootQueue = fs.getQueueManager().getRootQueue(); + + converter.convertQueue(rootQueue); + } + + @Test + public void testQueueMaxAMShare() { + converter.convertQueue(rootQueue); + + // root.admins.bob + assertEquals("root.admins.bob AM share", "1.0", + csConfig.get(PREFIX + "root.admins.bob.maximum-am-resource-percent")); + + // root.admins.alice + assertEquals("root.admins.alice AM share", "0.15", + csConfig.get(PREFIX + + "root.admins.alice.maximum-am-resource-percent")); + + Set remaining = Sets.difference(ALL_QUEUES, + Sets.newHashSet("root.admins.bob", "root.admins.alice")); + assertNoValueForQueues(remaining, ".maximum-am-resource-percent", + csConfig); + } + + @Test + public void testQueueMaxRunningApps() { + converter.convertQueue(rootQueue); + + assertEquals("root.admins.alice max apps", 2, + csConfig.getInt(PREFIX + "root.admins.alice.maximum-applications", + -1)); + + Set remaining = Sets.difference(ALL_QUEUES, + Sets.newHashSet("root.admins.alice")); + assertNoValueForQueues(remaining, ".maximum-applications", csConfig); + } + + @Test + public void testQueueMaxAllocations() { + converter.convertQueue(rootQueue); + + // root.admins vcores + mb + assertEquals("root.admins max vcores", 3, + csConfig.getInt(PREFIX + "root.admins.maximum-allocation-vcores", -1)); + assertEquals("root.admins max memory", 4096, + csConfig.getInt(PREFIX + "root.admins.maximum-allocation-mb", -1)); + + // root.users.john max vcores + mb + assertEquals("root.users.john max vcores", 2, + csConfig.getInt(PREFIX + "root.users.john.maximum-allocation-vcores", + -1)); + assertEquals("root.users.john max memory", 8192, + csConfig.getInt(PREFIX + "root.users.john.maximum-allocation-mb", -1)); + + Set remaining = Sets.difference(ALL_QUEUES, + Sets.newHashSet("root.admins", "root.users.john")); + assertNoValueForQueues(remaining, ".maximum-allocation-vcores", csConfig); + assertNoValueForQueues(remaining, ".maximum-allocation-mb", csConfig); + } + + @Test + public void testQueuePreemptionDisabled() { + converter = new FSQueueConverter(ruleHandler, + csConfig, + true, + false, + false, + CLUSTER_RESOURCE, + 0.16f, + 15); + + converter.convertQueue(rootQueue); + + assertTrue("root.admins.alice preemption setting", + csConfig.getBoolean(PREFIX + "root.admins.alice.disable_preemption", + false)); + assertTrue("root.users.joe preemption setting", + csConfig.getBoolean(PREFIX + "root.users.joe.disable_preemption", + false)); + + Set remaining = Sets.difference(ALL_QUEUES, + Sets.newHashSet("root.admins.alice", "root.users.joe")); + assertNoValueForQueues(remaining, ".disable_preemption", csConfig); + } + + @Test + public void testQueuePreemptionDisabledWhenGlobalPreemptionDisabled() { + converter.convertQueue(rootQueue); + + assertNoValueForQueues(ALL_QUEUES, ".disable_preemption", csConfig); + } + + @Test + public void testChildCapacity() { + converter.convertQueue(rootQueue); + + // root + assertEquals("root.default capacity", "33.333", + csConfig.get(PREFIX + "root.default.capacity")); + assertEquals("root.admins capacity", "33.333", + csConfig.get(PREFIX + "root.admins.capacity")); + assertEquals("root.users capacity", "66.667", + csConfig.get(PREFIX + "root.users.capacity")); + + // root.users + assertEquals("root.users.john capacity", "25.000", + csConfig.get(PREFIX + "root.users.john.capacity")); + assertEquals("root.users.joe capacity", "75.000", + csConfig.get(PREFIX + "root.users.joe.capacity")); + + // root.admins + assertEquals("root.admins.alice capacity", "75.000", + csConfig.get(PREFIX + "root.admins.alice.capacity")); + assertEquals("root.admins.bob capacity", "25.000", + csConfig.get(PREFIX + "root.admins.bob.capacity")); + } + + @Test + public void testQueueMaximumCapacity() { + converter.convertQueue(rootQueue); + + assertEquals("root.users.joe maximum capacity", "[memory=8192, vcores=8]", + csConfig.get(PREFIX + "root.users.joe.maximum-capacity")); + assertEquals("root.admins.bob maximum capacity", "[memory=8192, vcores=2]", + csConfig.get(PREFIX + "root.admins.bob.maximum-capacity")); + assertEquals("root.admins.alice maximum capacity", + "[memory=16384, vcores=4]", + csConfig.get(PREFIX + "root.admins.alice.maximum-capacity")); + + Set remaining = Sets.difference(ALL_QUEUES, + Sets.newHashSet("root.users.joe", + "root.admins.bob", + "root.admins.alice")); + assertNoValueForQueues(remaining, ".maximum-capacity", csConfig); + } + + @Test + public void testQueueAutoCreateChildQueue() { + config.setBoolean(FairSchedulerConfiguration.ALLOW_UNDECLARED_POOLS, true); + converter = new FSQueueConverter(ruleHandler, + csConfig, + false, + false, + true, + CLUSTER_RESOURCE, + 0.16f, + 15); + + converter.convertQueue(rootQueue); + + assertTrueForQueues(ALL_QUEUES, ".auto-create-child-queue.enabled", + csConfig); + } + + @Test + public void testQueueSizeBasedWeightEnabled() { + converter = new FSQueueConverter(ruleHandler, + csConfig, + false, + true, + false, + CLUSTER_RESOURCE, + 0.16f, + 15); + + converter.convertQueue(rootQueue); + + assertTrueForQueues(ALL_QUEUES, + ".ordering-policy.fair.enable-size-based-weight", csConfig); + } + + @Test + public void testQueueSizeBasedWeightDisabled() { + converter.convertQueue(rootQueue); + + assertNoValueForQueues(ALL_QUEUES, + ".ordering-policy.fair.enable-size-based-weight", csConfig); + } + + @Test + public void testQueueOrderingPolicy() throws Exception { + String absolutePath = + new File("src/test/resources/fair-scheduler-orderingpolicy.xml") + .getAbsolutePath(); + config.set(FairSchedulerConfiguration.ALLOCATION_FILE, + FILE_PREFIX + absolutePath); + fs.close(); + fs = createFairScheduler(); + rootQueue = fs.getQueueManager().getRootQueue(); + + converter.convertQueue(rootQueue); + + // root + assertEquals("root ordering policy", "fair", + csConfig.get(PREFIX + "root.ordering-policy")); + assertEquals("root.default ordering policy", "fair", + csConfig.get(PREFIX + "root.default.ordering-policy")); + assertEquals("root.admins ordering policy", "fair", + csConfig.get(PREFIX + "root.admins.ordering-policy")); + assertEquals("root.users ordering policy", "fair", + csConfig.get(PREFIX + "root.users.ordering-policy")); + + // root.users + assertEquals("root.users.joe ordering policy", "fair", + csConfig.get(PREFIX + "root.users.joe.ordering-policy")); + assertEquals("root.users.john ordering policy", "FIFO", + csConfig.get(PREFIX + "root.users.john.ordering-policy")); + + // root.admins + assertEquals("root.admins.alice ordering policy", "FIFO", + csConfig.get(PREFIX + "root.admins.alice.ordering-policy")); + assertEquals("root.admins.bob ordering policy", "fair", + csConfig.get(PREFIX + "root.admins.bob.ordering-policy")); + } + + @Test + public void testQueueMaxChildCapacityNotSupported() { + expectedException.expect(UnsupportedPropertyException.class); + expectedException.expectMessage("test"); + + Mockito.doThrow(new UnsupportedPropertyException("test")) + .when(ruleHandler).handleMaxChildCapacity(); + + converter.convertQueue(rootQueue); + } + + @Test + public void testReservationSystemNotSupported() { + expectedException.expect(UnsupportedPropertyException.class); + expectedException.expectMessage("maxCapacity"); + + Mockito.doThrow(new UnsupportedPropertyException("maxCapacity")) + .when(ruleHandler).handleMaxChildCapacity(); + config.setBoolean(YarnConfiguration.RM_RESERVATION_SYSTEM_ENABLE, true); + + converter.convertQueue(rootQueue); + } + + private void assertNoValueForQueues(Set queues, String postfix, + Configuration config) { + for (String queue : queues) { + String key = PREFIX + queue + postfix; + assertNull("Key " + key + " has value, but it should be null", + config.get(key)); + } + } + + private void assertTrueForQueues(Set queues, String postfix, + Configuration config) { + for (String queue : queues) { + String key = PREFIX + queue + postfix; + assertTrue("Key " + key + " is false, should be true", + config.getBoolean(key, false)); + } + } +} \ No newline at end of file diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/converter/TestFSYarnSiteConverter.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/converter/TestFSYarnSiteConverter.java new file mode 100644 index 00000000000..3822aea60ba --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/converter/TestFSYarnSiteConverter.java @@ -0,0 +1,142 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * http://www.apache.org/licenses/LICENSE-2.0 + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.converter; + +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.yarn.conf.YarnConfiguration; +import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacitySchedulerConfiguration; +import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.FairSchedulerConfiguration; +import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.converter.FSYarnSiteConverter; +import org.junit.Assert; +import org.junit.Before; +import org.junit.Test; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertTrue; + +/** + * Unit tests for FSYarnSiteConverter. + * + */ +public class TestFSYarnSiteConverter { + private Configuration yarnConfig; + private FSYarnSiteConverter converter; + private Configuration yarnConvertedConfig; + + @Before + public void setup() { + yarnConfig = new Configuration(false); + yarnConvertedConfig = new Configuration(false); + converter = new FSYarnSiteConverter(); + } + + @SuppressWarnings("deprecation") + @Test + public void testSiteContinuousSchedulingConversion() { + yarnConfig.setBoolean( + FairSchedulerConfiguration.CONTINUOUS_SCHEDULING_ENABLED, true); + yarnConfig.setInt( + FairSchedulerConfiguration.CONTINUOUS_SCHEDULING_SLEEP_MS, 666); + + converter.convertSiteProperties(yarnConfig, yarnConvertedConfig); + + assertTrue("Cont. scheduling", yarnConvertedConfig.getBoolean( + CapacitySchedulerConfiguration.SCHEDULE_ASYNCHRONOUSLY_ENABLE, false)); + assertEquals("Scheduling interval", 666, + yarnConvertedConfig.getInt( + "yarn.scheduler.capacity.schedule-asynchronously.scheduling-interval-ms", -1)); + } + + @Test + public void testSiteMinimumAllocationIncrementConversion() { + yarnConfig.setInt("yarn.resource-types.memory-mb.increment-allocation", 11); + yarnConfig.setInt("yarn.resource-types.vcores.increment-allocation", 5); + + converter.convertSiteProperties(yarnConfig, yarnConvertedConfig); + + assertEquals("Memory alloc increment", 11, + yarnConvertedConfig.getInt("yarn.scheduler.minimum-allocation-mb", + -1)); + assertEquals("Vcore increment", 5, + yarnConvertedConfig.getInt("yarn.scheduler.minimum-allocation-vcores", + -1)); + } + + @Test + public void testSitePreemptionConversion() { + yarnConfig.setBoolean(FairSchedulerConfiguration.PREEMPTION, true); + yarnConfig.setInt(FairSchedulerConfiguration.WAIT_TIME_BEFORE_KILL, 123); + yarnConfig.setInt( + FairSchedulerConfiguration.WAIT_TIME_BEFORE_NEXT_STARVATION_CHECK_MS, + 321); + + converter.convertSiteProperties(yarnConfig, yarnConvertedConfig); + + assertTrue("Preemption enabled", + yarnConvertedConfig.getBoolean( + YarnConfiguration.RM_SCHEDULER_ENABLE_MONITORS, + false)); + assertEquals("Wait time before kill", 123, + yarnConvertedConfig.getInt( + CapacitySchedulerConfiguration.PREEMPTION_WAIT_TIME_BEFORE_KILL, + -1)); + assertEquals("Starvation check wait time", 321, + yarnConvertedConfig.getInt( + CapacitySchedulerConfiguration.PREEMPTION_MONITORING_INTERVAL, + -1)); + } + + @Test + public void testSiteAssignMultipleConversion() { + yarnConfig.setBoolean(FairSchedulerConfiguration.ASSIGN_MULTIPLE, true); + + converter.convertSiteProperties(yarnConfig, yarnConvertedConfig); + + assertTrue("Assign multiple", + yarnConvertedConfig.getBoolean( + CapacitySchedulerConfiguration.ASSIGN_MULTIPLE_ENABLED, + false)); + } + + @Test + public void testSiteMaxAssignConversion() { + yarnConfig.setInt(FairSchedulerConfiguration.MAX_ASSIGN, 111); + + converter.convertSiteProperties(yarnConfig, yarnConvertedConfig); + + assertEquals("Max assign", 111, + yarnConvertedConfig.getInt( + CapacitySchedulerConfiguration.MAX_ASSIGN_PER_HEARTBEAT, -1)); + } + + @Test + public void testSiteLocalityThresholdConversion() { + yarnConfig.set(FairSchedulerConfiguration.LOCALITY_THRESHOLD_NODE, + "123.123"); + yarnConfig.set(FairSchedulerConfiguration.LOCALITY_THRESHOLD_RACK, + "321.321"); + + converter.convertSiteProperties(yarnConfig, yarnConvertedConfig); + + assertEquals("Locality threshold node", "123.123", + yarnConvertedConfig.get( + CapacitySchedulerConfiguration.NODE_LOCALITY_DELAY)); + assertEquals("Locality threshold rack", "321.321", + yarnConvertedConfig.get( + CapacitySchedulerConfiguration.RACK_LOCALITY_ADDITIONAL_DELAY)); + } +} diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/resources/conversion-rules.properties b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/resources/conversion-rules.properties new file mode 100644 index 00000000000..b3e421d8d73 --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/resources/conversion-rules.properties @@ -0,0 +1,24 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License" you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# http://www.apache.org/licenses/LICENSE-2.0 +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +maxCapacityPercentage.action=abort +maxChildCapacity.action=ABORT +userMaxRunningApps.action=abort +userMaxAppsDefault.action=ABORT +dynamicMaxAssign.action=abort +specifiedNotFirstRule.action=ABORT +reservationSystem.action=abort +queueAutoCreate.action=ABORT \ No newline at end of file diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/resources/fair-scheduler-conversion.xml b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/resources/fair-scheduler-conversion.xml new file mode 100644 index 00000000000..67f9ed9a904 --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/resources/fair-scheduler-conversion.xml @@ -0,0 +1,94 @@ + + + + + 1.0 + drf + alice,bob,joe,john hadoop_users + alice,bob,joe,john hadoop_users + + 1.0 + drf + + + 1.0 + drf + + 1.0 + drf + john + john + vcores=2,memory-mb=8192 + + + memory-mb=50.0%, vcores=50.0% + 3.0 + false + drf + joe + joe + + + + memory-mb=8192, vcores=1 + 1.0 + drf + vcores=3,memory-mb=4096 + + memory-mb=16384, vcores=4 + 2 + 3.0 + false + drf + alice + alice + 0.15 + memory-mb=16384, vcores=4 + + + memory-mb=8192, vcores=2 + 1.0 + drf + bob + bob + -1.0 + + + + + 30 + + 10 + 23 + 24 + 0.12 + 15 + fair + 0.16 + + + + + + + + + + + diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/resources/fair-scheduler-invalid.xml b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/resources/fair-scheduler-invalid.xml new file mode 100644 index 00000000000..4afed0e8e4c --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/resources/fair-scheduler-invalid.xml @@ -0,0 +1,21 @@ + + + + + \ No newline at end of file diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/resources/fair-scheduler-max-resources-percentage.xml b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/resources/fair-scheduler-max-resources-percentage.xml new file mode 100644 index 00000000000..a286ac89c65 --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/resources/fair-scheduler-max-resources-percentage.xml @@ -0,0 +1,90 @@ + + + + + 1.0 + drf + alice,bob,joe,john hadoop_users + alice,bob,joe,john hadoop_users + + 1.0 + drf + + + 1.0 + drf + + 1.0 + drf + john + john + vcores=2,memory-mb=8192 + + + memory-mb=50.0%, vcores=50.0% + 3.0 + false + drf + joe + joe + + + + memory-mb=8192, vcores=1 + 1.0 + drf + vcores=3,memory-mb=4096 + + memory-mb=16384, vcores=4 + 2 + 3.0 + false + drf + alice + alice + 0.15 + memory-mb=16384, vcores=4 + + + memory-mb=8192, vcores=2 + 1.0 + drf + bob + bob + -1.0 + + + + 23 + 24 + 0.12 + 15 + fair + 0.16 + + + + + + + + + + + diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/resources/fair-scheduler-orderingpolicy-mixed.xml b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/resources/fair-scheduler-orderingpolicy-mixed.xml new file mode 100644 index 00000000000..3a2a59300cd --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/resources/fair-scheduler-orderingpolicy-mixed.xml @@ -0,0 +1,89 @@ + + + + + 1.0 + drf + alice,bob,joe,john hadoop_users + alice,bob,joe,john hadoop_users + + 1.0 + fair + + + 1.0 + drf + + 1.0 + drf + john + john + vcores=2,memory-mb=8192 + + + memory-mb=50.0%, vcores=50.0% + 3.0 + false + drf + joe + joe + + + + memory-mb=8192, vcores=1 + 1.0 + fair + vcores=3,memory-mb=4096 + + memory-mb=16384, vcores=4 + 2 + 3.0 + false + drf + alice + alice + 0.15 + + + memory-mb=8192, vcores=2 + 1.0 + drf + bob + bob + -1.0 + + + + 23 + 24 + 0.12 + 15 + drf + 0.16 + + + + + + + + + + + diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/resources/fair-scheduler-orderingpolicy.xml b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/resources/fair-scheduler-orderingpolicy.xml new file mode 100644 index 00000000000..1b619c21601 --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/resources/fair-scheduler-orderingpolicy.xml @@ -0,0 +1,89 @@ + + + + + 1.0 + fair + alice,bob,joe,john hadoop_users + alice,bob,joe,john hadoop_users + + 1.0 + fair + + + 1.0 + fair + + 1.0 + fifo + john + john + vcores=2,memory-mb=8192 + + + memory-mb=50.0%, vcores=50.0% + 3.0 + false + fair + joe + joe + + + + memory-mb=8192, vcores=1 + 1.0 + fair + vcores=3,memory-mb=4096 + + memory-mb=16384, vcores=4 + 2 + 3.0 + false + fifo + alice + alice + 0.15 + + + memory-mb=8192, vcores=2 + 1.0 + fair + bob + bob + -1.0 + + + + 23 + 24 + 0.12 + 15 + fair + 0.16 + + + + + + + + + + + diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/resources/fair-scheduler-sameleafqueue.xml b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/resources/fair-scheduler-sameleafqueue.xml new file mode 100644 index 00000000000..d28e7b91946 --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/resources/fair-scheduler-sameleafqueue.xml @@ -0,0 +1,90 @@ + + + + + 1.0 + drf + alice,bob,joe,john hadoop_users + alice,bob,joe,john hadoop_users + + 1.0 + drf + + + 1.0 + drf + + 1.0 + drf + john + john + vcores=2,memory-mb=8192 + + + memory-mb=50.0%, vcores=50.0% + 3.0 + false + drf + joe + joe + + + + memory-mb=8192, vcores=1 + 1.0 + drf + vcores=3,memory-mb=4096 + + memory-mb=16384, vcores=4 + 2 + 3.0 + false + drf + alice + alice + 0.15 + memory-mb=16384, vcores=4 + + + memory-mb=8192, vcores=2 + 1.0 + drf + bob + bob + -1.0 + + + + 23 + 24 + 0.12 + 15 + fair + 0.16 + + + + + + + + + + + diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/resources/yarn-site-with-allocation-file-ref.xml b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/resources/yarn-site-with-allocation-file-ref.xml new file mode 100644 index 00000000000..5352d434517 --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/resources/yarn-site-with-allocation-file-ref.xml @@ -0,0 +1,23 @@ + + + + + + yarn.scheduler.fair.allocation.file + fair-scheduler-conversion.xml + + \ No newline at end of file diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/resources/yarn-site-with-invalid-allocation-file-ref.xml b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/resources/yarn-site-with-invalid-allocation-file-ref.xml new file mode 100644 index 00000000000..248040d928c --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/resources/yarn-site-with-invalid-allocation-file-ref.xml @@ -0,0 +1,37 @@ + + + + + + + + + yarn.scheduler.fair.allocation.file + + \ No newline at end of file diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/YarnCommands.md b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/YarnCommands.md index e17538ccdfc..164e5532ce9 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/YarnCommands.md +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/YarnCommands.md @@ -193,6 +193,7 @@ Usage: `yarn resourcemanager [-format-state-store]` |:---- |:---- | | -format-state-store | Formats the RMStateStore. This will clear the RMStateStore and is useful if past applications are no longer needed. This should be run only when the ResourceManager is not running. | | -remove-application-from-state-store \ | Remove the application from RMStateStore. This should be run only when the ResourceManager is not running. | +| -convert-fs-configuration [-y|yarnsiteconfig] [-f|fsconfig] [-r|rulesconfig] [-o|output-directory] [-p|print] [-c|cluster-resource] | Converts the specified Fair Scheduler configuration to Capacity Scheduler configuration. Requires two mandatory input files. First, the yarn-site.xml with the following format: [-y|yarnsiteconfig [\]. Secondly, the fair-scheduler.xml with the following format: [-f|fsconfig [\]. This config is not mandatory if there is a reference in yarn-site.xml to the fair-scheduler.xml with the property 'yarn.scheduler.fair.allocation.file'. If both are defined, the -f option has precedence. The output directory of the config files should be specified as well, with: \[-o|output-directory\ \]. An optional rules config file could be also specified with the following format: [-r|rulesconfig \]. The rule config file's format is a property file. There's an additional \[-p|print\] parameter, which is optional. If defined, the configuration will be emitted to the console instead. In its normal operation, the output files (yarn-site.xml and capacity-scheduler.xml) of this command is generated to the specified output directory. The cluster resource parameter (\[-c|cluster-resource\] \\]) needs to be specified if any queue has a maxResources setting with value as percentage. The format of the resource string is the same as in fair-scheduler.xml.) ] | Start the ResourceManager