diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java index 51731f3..9fa99a3 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java @@ -535,7 +535,9 @@ * Adjustment to make to the container os scheduling priority. * The valid values for this could vary depending on the platform. * On Linux, higher values mean run the containers at a less - * favorable priority than the NM. + * favorable priority than the NM. + * On Window the values are 32 (NORMAL), 64 (LOW), 128 (HIGH), 256 (REALTIME), + * 16384 (BELOWNORMAL) and 32768 (ABOVENORMAL). * The value specified is an int. */ public static final String NM_CONTAINER_EXECUTOR_SCHED_PRIORITY = diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml index c94b782..3af1b9a 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml @@ -636,6 +636,14 @@ org.apache.hadoop.yarn.server.nodemanager.DefaultContainerExecutor + + + Adjustment to make to the container os scheduling priority. The valid values for this could vary depending on the platform. +On Linux, higher values mean run the containers at a less favorable priority than the NM. +On Window the values are 32 (NORMAL), 64 (LOW), 128 (HIGH), 256 (REALTIME), 16384 (BELOWNORMAL) and 32768 (ABOVENORMAL). +The value specified is an int. + yarn.nodemanager.container-executor.os.sched.priority.adjustment + Number of threads container manager uses. diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/ContainerExecutor.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/ContainerExecutor.java index b76b17d..1cba905 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/ContainerExecutor.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/ContainerExecutor.java @@ -58,10 +58,19 @@ private ReentrantReadWriteLock lock = new ReentrantReadWriteLock(); private final ReadLock readLock = lock.readLock(); private final WriteLock writeLock = lock.writeLock(); - + + protected boolean containerSchedPriorityIsSet = false; + protected int containerSchedPriorityAdjustment = 0; + @Override public void setConf(Configuration conf) { this.conf = conf; + if (conf.get(YarnConfiguration.NM_CONTAINER_EXECUTOR_SCHED_PRIORITY) != null) { + containerSchedPriorityIsSet = true; + containerSchedPriorityAdjustment = conf + .getInt(YarnConfiguration.NM_CONTAINER_EXECUTOR_SCHED_PRIORITY, + YarnConfiguration.DEFAULT_NM_CONTAINER_EXECUTOR_SCHED_PRIORITY); + } } @Override diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/LinuxContainerExecutor.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/LinuxContainerExecutor.java index cbdcb13..97865fc 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/LinuxContainerExecutor.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/LinuxContainerExecutor.java @@ -54,8 +54,6 @@ private Pattern nonsecureLocalUserPattern; private String containerExecutorExe; private LCEResourcesHandler resourcesHandler; - private boolean containerSchedPriorityIsSet = false; - private int containerSchedPriorityAdjustment = 0; @Override @@ -68,12 +66,6 @@ public void setConf(Configuration conf) { DefaultLCEResourcesHandler.class, LCEResourcesHandler.class), conf); resourcesHandler.setConf(conf); - if (conf.get(YarnConfiguration.NM_CONTAINER_EXECUTOR_SCHED_PRIORITY) != null) { - containerSchedPriorityIsSet = true; - containerSchedPriorityAdjustment = conf - .getInt(YarnConfiguration.NM_CONTAINER_EXECUTOR_SCHED_PRIORITY, - YarnConfiguration.DEFAULT_NM_CONTAINER_EXECUTOR_SCHED_PRIORITY); - } nonsecureLocalUser = conf.get( YarnConfiguration.NM_NONSECURE_MODE_LOCAL_USER_KEY, YarnConfiguration.DEFAULT_NM_NONSECURE_MODE_LOCAL_USER); diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/WindowsSecureContainerExecutor.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/WindowsSecureContainerExecutor.java index 40c79fc..cf2da36 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/WindowsSecureContainerExecutor.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/WindowsSecureContainerExecutor.java @@ -22,6 +22,7 @@ import java.io.PrintStream; import java.net.InetSocketAddress; import java.util.ArrayList; +import java.util.Arrays; import java.util.HashMap; import java.util.List; import java.util.Map; @@ -36,6 +37,7 @@ import org.apache.hadoop.util.Shell.ShellCommandExecutor; import org.apache.hadoop.yarn.conf.YarnConfiguration; import org.apache.hadoop.yarn.server.nodemanager.DefaultContainerExecutor.LocalWrapperScriptBuilder; +import org.apache.hadoop.yarn.server.nodemanager.containermanager.launcher.WindowsLauncherPriorityUtil; import org.apache.hadoop.yarn.server.nodemanager.containermanager.localizer.ContainerLocalizer; /** @@ -61,18 +63,23 @@ protected void writeLocalWrapperScript(Path launchDst, Path pidFile, PrintStream } private String nodeManagerGroup; + + private WindowsLauncherPriorityUtil launchPriority = null; @Override public void setConf(Configuration conf) { super.setConf(conf); nodeManagerGroup = conf.get(YarnConfiguration.NM_WINDOWS_SECURE_CONTAINER_GROUP); + launchPriority = WindowsLauncherPriorityUtil.getInstance(conf); } @Override protected String[] getRunCommand(String command, String groupId, String userName, Path pidFile, Configuration conf) { - return new String[] { Shell.WINUTILS, "task", "createAsUser", groupId, userName, - pidFile.toString(), "cmd /c " + command }; + List commands = new ArrayList(); + commands.addAll(Arrays.asList(Shell.WINUTILS, "task", "createAsUser", groupId, userName, + pidFile.toString(), "cmd /c " + command)); + return commands.toArray(new String[commands.size()]); } @Override @@ -105,6 +112,7 @@ public void localizeClasspathJar(Path classpathJar, String owner) throws IOExcep lfs.setOwner(classpathJar, owner, nodeManagerGroup); } + @Override public void startLocalizer(Path nmPrivateContainerTokens, InetSocketAddress nmAddr, String user, String appId, String locId, @@ -132,6 +140,8 @@ public void startLocalizer(Path nmPrivateContainerTokens, LOG.info(String.format("cwdApp: %s", cwdApp)); command = new ArrayList(); + + command = launchPriority.addSchedPriorityCommand(command); command.add(Shell.WINUTILS); command.add("task"); @@ -169,6 +179,8 @@ public void startLocalizer(Path nmPrivateContainerTokens, command.add(dir); } commandArray = command.toArray(new String[command.size()]); + + shExec = new ShellCommandExecutor( commandArray, cwdApp); diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/launcher/ContainerLaunch.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/launcher/ContainerLaunch.java index 174e932..b2de0c4 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/launcher/ContainerLaunch.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/launcher/ContainerLaunch.java @@ -77,6 +77,7 @@ import org.apache.hadoop.yarn.util.ConverterUtils; import com.google.common.annotations.VisibleForTesting; +import com.google.common.collect.Lists; public class ContainerLaunch implements Callable { @@ -268,7 +269,7 @@ public Integer call() { localResources); // Write out the environment - writeLaunchEnv(containerScriptOutStream, environment, localResources, + writeLaunchEnv(conf, containerScriptOutStream, environment, localResources, launchContext.getCommands()); // /////////// End of writing out container-script @@ -481,8 +482,8 @@ private String getAppPrivateDir(String appIdStr) { @VisibleForTesting static abstract class ShellScriptBuilder { - public static ShellScriptBuilder create() { - return Shell.WINDOWS ? new WindowsShellScriptBuilder() : + public static ShellScriptBuilder create(Configuration conf) { + return Shell.WINDOWS ? new WindowsShellScriptBuilder(conf) : new UnixShellScriptBuilder(); } @@ -569,6 +570,8 @@ protected void mkdir(Path path) { private static final class WindowsShellScriptBuilder extends ShellScriptBuilder { + + private final WindowsLauncherPriorityUtil launcherPriority; private void errorCheck() { line("@if %errorlevel% neq 0 exit /b %errorlevel%"); @@ -579,13 +582,15 @@ private void lineWithLenCheck(String... commands) throws IOException { line(commands); } - public WindowsShellScriptBuilder() { + public WindowsShellScriptBuilder(Configuration conf) { + launcherPriority = WindowsLauncherPriorityUtil.getInstance(conf); line("@setlocal"); line(); } @Override public void command(List command) throws IOException { + command = launcherPriority.addSchedPriorityCommand(command); lineWithLenCheck("@call ", StringUtils.join(" ", command)); errorCheck(); } @@ -758,11 +763,11 @@ public void sanitizeEnv(Map environment, Path pwd, } } - static void writeLaunchEnv(OutputStream out, + static void writeLaunchEnv(Configuration conf, OutputStream out, Map environment, Map> resources, List command) throws IOException { - ShellScriptBuilder sb = ShellScriptBuilder.create(); + ShellScriptBuilder sb = ShellScriptBuilder.create(conf); if (environment != null) { for (Map.Entry env : environment.entrySet()) { sb.env(env.getKey().toString(), env.getValue().toString()); diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/launcher/WindowsLauncherPriorityUtil.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/launcher/WindowsLauncherPriorityUtil.java new file mode 100644 index 0000000..471235d --- /dev/null +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/launcher/WindowsLauncherPriorityUtil.java @@ -0,0 +1,126 @@ +/** + * + */ +package org.apache.hadoop.yarn.server.nodemanager.containermanager.launcher; + +import java.util.ArrayList; +import java.util.List; + +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.apache.hadoop.conf.Configurable; +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.yarn.conf.YarnConfiguration; + +public class WindowsLauncherPriorityUtil implements Configurable{ + + private static final Log LOG = LogFactory + .getLog(WindowsLauncherPriorityUtil.class); + + // These are the NT API priority levels + private final static int NORMAL_PRIORITY_CLASS = 0x00000020; + private final static int IDLE_PRIORITY_CLASS = 0x0000040; + private final static int HIGH_PRIORITY_CLASS = 0x00000080; + private final static int REALTIME_PRIORITY_CLASS = 0x00000100; + private final static int BELOW_NORMAL_PRIORITY_CLASS = 0x00004000; + private final static int ABOVE_NORMAL_PRIORITY_CLASS = 0x00008000; + + //These are the start.exe command line priority levels + private final static String NORMAL_PRIORITY = "NORMAL"; + private final static String IDLE_PRIORITY = "LOW"; + private final static String HIGH_PRIORITY = "HIGH"; + private final static String REALTIME_PRIORITY = "REALTIME"; + private final static String ABOVENORMAL_PRIORITY = "ABOVENORMAL"; + private final static String BELOWNORMAL_PRIORITY = "BELOWNORMAL"; + + private static WindowsLauncherPriorityUtil theInstance = null; + + public static synchronized WindowsLauncherPriorityUtil getInstance(Configuration conf) { + if (theInstance == null) { + theInstance = new WindowsLauncherPriorityUtil(); + theInstance.setConf(conf); + } + return theInstance; + } + + private transient Configuration conf; + + private boolean containerSchedPriorityIsSet = false; + + private String startPriorityLevel = null; + + @Override + public void setConf(Configuration conf) { + this.conf = conf; + int containerSchedPriorityAdjustment = 0; + if (conf.get(YarnConfiguration.NM_CONTAINER_EXECUTOR_SCHED_PRIORITY) != null) { + containerSchedPriorityIsSet = true; + containerSchedPriorityAdjustment = conf + .getInt(YarnConfiguration.NM_CONTAINER_EXECUTOR_SCHED_PRIORITY, + YarnConfiguration.DEFAULT_NM_CONTAINER_EXECUTOR_SCHED_PRIORITY); + } + if (containerSchedPriorityIsSet) { + switch(containerSchedPriorityAdjustment ) { + case NORMAL_PRIORITY_CLASS: + startPriorityLevel = NORMAL_PRIORITY; + break; + case IDLE_PRIORITY_CLASS: + startPriorityLevel = IDLE_PRIORITY; + break; + case HIGH_PRIORITY_CLASS: + startPriorityLevel = HIGH_PRIORITY; + break; + case REALTIME_PRIORITY_CLASS: + startPriorityLevel = REALTIME_PRIORITY; + break; + case BELOW_NORMAL_PRIORITY_CLASS: + startPriorityLevel = BELOWNORMAL_PRIORITY; + break; + case ABOVE_NORMAL_PRIORITY_CLASS: + startPriorityLevel = ABOVENORMAL_PRIORITY; + break; + default: + LOG.warn(String.format("An invalid priority level %d was set for %s. Valid values are:" + + "%d (%s), %d (%s), %d (%s), %d (%s) and %d (%s)." + + "No priority adjustment will be used for container launch.", + containerSchedPriorityAdjustment, YarnConfiguration.NM_CONTAINER_EXECUTOR_SCHED_PRIORITY, + NORMAL_PRIORITY_CLASS, NORMAL_PRIORITY, + IDLE_PRIORITY_CLASS, IDLE_PRIORITY, + HIGH_PRIORITY_CLASS, HIGH_PRIORITY, + REALTIME_PRIORITY_CLASS, REALTIME_PRIORITY, + BELOW_NORMAL_PRIORITY_CLASS, BELOWNORMAL_PRIORITY, + ABOVE_NORMAL_PRIORITY_CLASS, ABOVENORMAL_PRIORITY)); + containerSchedPriorityIsSet = false; + break; + } + if (containerSchedPriorityIsSet) { + LOG.info(String.format("Priority adjustment set to %d (%s)", + containerSchedPriorityAdjustment, startPriorityLevel)); + } + } + } + + /*(nojavadoc) + * Adds the start /HIGH /wait in front of the command. This adjusts the priority. + * /WAIT will cause it stop until command executes + */ + public List addSchedPriorityCommand(List command) { + if (containerSchedPriorityIsSet) { + List newCommand = new ArrayList(command.size() + 6); + // start is an internal cmd.exe command + newCommand.add("cmd.exe"); + newCommand.add("/C"); + newCommand.add("start"); + newCommand.add("/" + startPriorityLevel); + newCommand.add("/WAIT"); + newCommand.addAll(command); + command = newCommand; + } + return command; + } + + @Override + public Configuration getConf() { + return this.conf; + } +} diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/launcher/TestContainerLaunch.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/launcher/TestContainerLaunch.java index c8fc85a..2e6e036 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/launcher/TestContainerLaunch.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/launcher/TestContainerLaunch.java @@ -141,7 +141,7 @@ public void testSpecialCharSymlinks() throws IOException { commands.add("/bin/sh ./\\\"" + badSymlink + "\\\""); } - ContainerLaunch.writeLaunchEnv(fos, env, resources, commands); + ContainerLaunch.writeLaunchEnv(conf, fos, env, resources, commands); fos.flush(); fos.close(); FileUtil.setExecutable(tempFile, true); @@ -208,7 +208,7 @@ public void testInvalidSymlinkDiagnostics() throws IOException { } else { commands.add("/bin/sh ./\\\"" + symLink + "\\\""); } - ContainerLaunch.writeLaunchEnv(fos, env, resources, commands); + ContainerLaunch.writeLaunchEnv(conf, fos, env, resources, commands); fos.flush(); fos.close(); FileUtil.setExecutable(tempFile, true); @@ -261,7 +261,7 @@ public void testInvalidEnvSyntaxDiagnostics() throws IOException { "\"workflowName\":\"\n\ninsert table " + "\npartition (cd_education_status)\nselect cd_demo_sk, cd_gender, " ); List commands = new ArrayList(); - ContainerLaunch.writeLaunchEnv(fos, env, resources, commands); + ContainerLaunch.writeLaunchEnv(conf, fos, env, resources, commands); fos.flush(); fos.close(); @@ -338,7 +338,7 @@ public void testContainerLaunchStdoutAndStderrDiagnostics() throws IOException { Map env = new HashMap(); List commands = new ArrayList(); commands.add(command); - ContainerLaunch.writeLaunchEnv(fos, env, resources, commands); + ContainerLaunch.writeLaunchEnv(conf, fos, env, resources, commands); fos.flush(); fos.close(); @@ -803,7 +803,7 @@ protected Token createContainerToken(ContainerId cId) throws InvalidToken { */ @Test (timeout = 10000) public void testShellScriptBuilderNonZeroExitCode() throws IOException { - ShellScriptBuilder builder = ShellScriptBuilder.create(); + ShellScriptBuilder builder = ShellScriptBuilder.create(conf); builder.command(Arrays.asList(new String[] {"unknownCommand"})); File shellFile = Shell.appendScriptExtension(tmpDir, "testShellScriptBuilderError"); PrintStream writer = new PrintStream(new FileOutputStream(shellFile)); @@ -840,7 +840,7 @@ public void testWindowsShellScriptBuilderCommand() throws IOException { // The tests are built on assuming 8191 max command line length assertEquals(8191, Shell.WINDOWS_MAX_SHELL_LENGHT); - ShellScriptBuilder builder = ShellScriptBuilder.create(); + ShellScriptBuilder builder = ShellScriptBuilder.create(conf); // Basic tests: less length, exact length, max+1 length builder.command(Arrays.asList( @@ -888,7 +888,7 @@ public void testWindowsShellScriptBuilderEnv() throws IOException { // The tests are built on assuming 8191 max command line length assertEquals(8191, Shell.WINDOWS_MAX_SHELL_LENGHT); - ShellScriptBuilder builder = ShellScriptBuilder.create(); + ShellScriptBuilder builder = ShellScriptBuilder.create(conf); // test env builder.env("somekey", org.apache.commons.lang.StringUtils.repeat("A", 1024)); @@ -913,7 +913,7 @@ public void testWindowsShellScriptBuilderMkdir() throws IOException { // The tests are built on assuming 8191 max command line length assertEquals(8191, Shell.WINDOWS_MAX_SHELL_LENGHT); - ShellScriptBuilder builder = ShellScriptBuilder.create(); + ShellScriptBuilder builder = ShellScriptBuilder.create(conf); // test mkdir builder.mkdir(new Path(org.apache.commons.lang.StringUtils.repeat("A", 1024))); @@ -938,7 +938,7 @@ public void testWindowsShellScriptBuilderLink() throws IOException { // The tests are built on assuming 8191 max command line length assertEquals(8191, Shell.WINDOWS_MAX_SHELL_LENGHT); - ShellScriptBuilder builder = ShellScriptBuilder.create(); + ShellScriptBuilder builder = ShellScriptBuilder.create(conf); // test link builder.link(new Path(org.apache.commons.lang.StringUtils.repeat("A", 1024)),