diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/Apps.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/Apps.java index 685c6d30540..1c90d551b7b 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/Apps.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/Apps.java @@ -23,6 +23,7 @@ import static org.apache.hadoop.yarn.util.StringHelper.sjoin; import java.io.File; +import java.util.ArrayList; import java.util.Iterator; import java.util.Map; import java.util.regex.Matcher; @@ -105,7 +106,26 @@ public static void setEnvFromInputString(Map env, } } } - + + /** + * + * @param envString String containing env variable definitions + * @param classPathSeparator String that separates the definitions + * @return ArrayList of environment variable names + */ + public static ArrayList getEnvVarsFromInputString(String envString, + String classPathSeparator) { + ArrayList envList = new ArrayList<>(); + if (envString != null && envString.length() > 0) { + Matcher varValMatcher = VARVAL_SPLITTER.matcher(envString); + while (varValMatcher.find()) { + String envVar = varValMatcher.group(1); + envList.add(envVar); + } + } + return envList; + } + /** * This older version of this method is kept around for compatibility * because downstream frameworks like Spark and Tez have been using it. diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/AuxiliaryServiceHelper.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/AuxiliaryServiceHelper.java index cb118f56da9..1374d96f261 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/AuxiliaryServiceHelper.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/AuxiliaryServiceHelper.java @@ -45,7 +45,7 @@ public static void setServiceDataIntoEnv(String serviceName, Base64.encodeBase64String(byteData)); } - private static String getPrefixServiceName(String serviceName) { + public static String getPrefixServiceName(String serviceName) { return NM_AUX_SERVICE + serviceName; } } diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/ContainerExecutor.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/ContainerExecutor.java index f4279a3b09e..f5d2a189728 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/ContainerExecutor.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/ContainerExecutor.java @@ -27,6 +27,7 @@ import java.util.ArrayList; import java.util.Arrays; import java.util.List; +import java.util.LinkedHashSet; import java.util.Map; import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.ConcurrentMap; @@ -316,14 +317,15 @@ public int reacquireContainer(ContainerReacquisitionContext ctx) * @param command the command that will be run * @param logDir the log dir to which to copy debugging information * @param user the username of the job owner + * @param nmVars the set of environment vars that are explicitly set by NM * @throws IOException if any errors happened writing to the OutputStream, * while creating symlinks */ public void writeLaunchEnv(OutputStream out, Map environment, Map> resources, List command, Path logDir, - String user) throws IOException { + String user, LinkedHashSet nmVars) throws IOException { this.writeLaunchEnv(out, environment, resources, command, logDir, user, - ContainerLaunch.CONTAINER_SCRIPT); + ContainerLaunch.CONTAINER_SCRIPT, nmVars); } /** @@ -339,14 +341,15 @@ public void writeLaunchEnv(OutputStream out, Map environment, * @param logDir the log dir to which to copy debugging information * @param user the username of the job owner * @param outFilename the path to which to write the launch environment + * @param nmVars the set of environment vars that are explicitly set by NM * @throws IOException if any errors happened writing to the OutputStream, * while creating symlinks */ @VisibleForTesting public void writeLaunchEnv(OutputStream out, Map environment, Map> resources, List command, Path logDir, - String user, String outFilename) throws IOException { - updateEnvForWhitelistVars(environment); + String user, String outFilename, LinkedHashSet nmVars) + throws IOException { ContainerLaunch.ShellScriptBuilder sb = ContainerLaunch.ShellScriptBuilder.create(); @@ -361,8 +364,42 @@ public void writeLaunchEnv(OutputStream out, Map environment, if (environment != null) { sb.echo("Setting up env variables"); + // Whitelist environment variables are treated specially. + // Only add them if they are not already defined in the environment. + // Add them using special syntax to prevent them from eclipsing + // variables that may be set explicitly in the container image (e.g, + // in a docker image). Put these before the others to ensure the + // correct expansion is used. + for(String var : whitelistVars) { + if (!environment.containsKey(var)) { + String val = getNMEnvVar(var); + if (val != null) { + sb.whitelistedEnv(var, val); + } + } + } + // Now write vars that were set explicitly by nodemanager, preserving + // the order they were written in. + if (nmVars != null) { + for (String nmEnvVar : nmVars) { + sb.env(nmEnvVar, environment.get(nmEnvVar)); + } + } + // Now write the remaining environment variables. for (Map.Entry env : environment.entrySet()) { - sb.env(env.getKey(), env.getValue()); + if (nmVars == null || !nmVars.contains(env.getKey())) { + sb.env(env.getKey(), env.getValue()); + } + } + // Add the whitelist vars to the environment. Do this after writing + // environment variables so they are not written twice. + for(String var : whitelistVars) { + if (!environment.containsKey(var)) { + String val = getNMEnvVar(var); + if (val != null) { + environment.put(var, val); + } + } } } @@ -663,23 +700,6 @@ protected boolean isContainerActive(ContainerId containerId) { } } - /** - * Propagate variables from the nodemanager's environment into the - * container's environment if unspecified by the container. - * @param env the environment to update - * @see org.apache.hadoop.yarn.conf.YarnConfiguration#NM_ENV_WHITELIST - */ - protected void updateEnvForWhitelistVars(Map env) { - for(String var : whitelistVars) { - if (!env.containsKey(var)) { - String val = getNMEnvVar(var); - if (val != null) { - env.put(var, val); - } - } - } - } - @VisibleForTesting protected String getNMEnvVar(String varname) { return System.getenv(varname); diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/LinuxContainerExecutor.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/LinuxContainerExecutor.java index d359f3159b6..f0b9797316f 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/LinuxContainerExecutor.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/LinuxContainerExecutor.java @@ -66,7 +66,6 @@ import java.util.ArrayList; import java.util.Arrays; import java.util.List; -import java.util.Map; import java.util.regex.Pattern; import static org.apache.hadoop.yarn.server.nodemanager.containermanager.linux.runtime.LinuxContainerRuntimeConstants.*; @@ -472,13 +471,6 @@ public void prepareContainer(ContainerPrepareContext ctx) throws IOException { } } - @Override - protected void updateEnvForWhitelistVars(Map env) { - if (linuxContainerRuntime.useWhitelistEnv(env)) { - super.updateEnvForWhitelistVars(env); - } - } - @Override public int launchContainer(ContainerStartContext ctx) throws IOException, ConfigurationException { diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/launcher/ContainerLaunch.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/launcher/ContainerLaunch.java index 7f43458d168..d054660b552 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/launcher/ContainerLaunch.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/launcher/ContainerLaunch.java @@ -33,6 +33,7 @@ import java.util.EnumSet; import java.util.HashMap; import java.util.List; +import java.util.LinkedHashSet; import java.util.Map; import java.util.Map.Entry; import java.util.concurrent.Callable; @@ -211,6 +212,9 @@ public Integer call() { } // /////////////////////////// End of variable expansion + // Use this to track variables that are added to the environment by nm. + LinkedHashSet nmEnvVars = new LinkedHashSet(); + FileContext lfs = FileContext.getLocalFSFileContext(); Path nmPrivateContainerScriptPath = dirsHandler.getLocalPathForWrite( @@ -256,6 +260,7 @@ public Integer call() { ApplicationConstants.CONTAINER_TOKEN_FILE_ENV_NAME, new Path(containerWorkDir, FINAL_CONTAINER_TOKENS_FILE).toUri().getPath()); + nmEnvVars.add(ApplicationConstants.CONTAINER_TOKEN_FILE_ENV_NAME); // /////////// Write out the container-script in the nmPrivate space. try (DataOutputStream containerScriptOutStream = @@ -263,14 +268,14 @@ public Integer call() { EnumSet.of(CREATE, OVERWRITE))) { // Sanitize the container's environment sanitizeEnv(environment, containerWorkDir, appDirs, userLocalDirs, - containerLogDirs, localResources, nmPrivateClasspathJarDir); + containerLogDirs, localResources, nmPrivateClasspathJarDir, nmEnvVars); prepareContainer(localResources, containerLocalDirs); // Write out the environment exec.writeLaunchEnv(containerScriptOutStream, environment, localResources, launchContext.getCommands(), - containerLogDir, user); + containerLogDir, user, nmEnvVars); } // /////////// End of writing out container-script @@ -1135,6 +1140,9 @@ public final void stderr(Path stderrDir, String stdErrFile) throws IOException { public abstract void env(String key, String value) throws IOException; + public abstract void whitelistedEnv(String key, String value) + throws IOException; + public abstract void echo(String echoStr) throws IOException; public final void symlink(Path src, Path dst) throws IOException { @@ -1254,6 +1262,11 @@ public void env(String key, String value) throws IOException { line("export ", key, "=\"", value, "\""); } + @Override + public void whitelistedEnv(String key, String value) throws IOException { + line("export ", key, "=${", key, ":-", "\"", value, "\"}"); + } + @Override public void echo(final String echoStr) throws IOException { line("echo \"" + echoStr + "\""); @@ -1344,6 +1357,11 @@ public void env(String key, String value) throws IOException { errorCheck(); } + @Override + public void whitelistedEnv(String key, String value) throws IOException { + env(key, value); + } + @Override public void echo(final String echoStr) throws IOException { lineWithLenCheck("@echo \"", echoStr, "\""); @@ -1402,37 +1420,46 @@ private static void putEnvIfAbsent( public void sanitizeEnv(Map environment, Path pwd, List appDirs, List userLocalDirs, List - containerLogDirs, - Map> resources, - Path nmPrivateClasspathJarDir) throws IOException { + containerLogDirs, Map> resources, + Path nmPrivateClasspathJarDir, + LinkedHashSet nmVars) throws IOException { /** * Non-modifiable environment variables */ environment.put(Environment.CONTAINER_ID.name(), container .getContainerId().toString()); + nmVars.add(Environment.CONTAINER_ID.name()); environment.put(Environment.NM_PORT.name(), String.valueOf(this.context.getNodeId().getPort())); + nmVars.add(Environment.NM_PORT.name()); environment.put(Environment.NM_HOST.name(), this.context.getNodeId() .getHost()); + nmVars.add(Environment.NM_HOST.name()); environment.put(Environment.NM_HTTP_PORT.name(), String.valueOf(this.context.getHttpPort())); + nmVars.add(Environment.NM_HTTP_PORT.name()); environment.put(Environment.LOCAL_DIRS.name(), StringUtils.join(",", appDirs)); + nmVars.add(Environment.LOCAL_DIRS.name()); environment.put(Environment.LOCAL_USER_DIRS.name(), StringUtils.join(",", userLocalDirs)); + nmVars.add(Environment.LOCAL_USER_DIRS.name()); environment.put(Environment.LOG_DIRS.name(), StringUtils.join(",", containerLogDirs)); + nmVars.add(Environment.LOG_DIRS.name()); environment.put(Environment.USER.name(), container.getUser()); - + nmVars.add(Environment.USER.name()); + environment.put(Environment.LOGNAME.name(), container.getUser()); + nmVars.add(Environment.LOGNAME.name()); environment.put(Environment.HOME.name(), conf.get( @@ -1440,37 +1467,44 @@ public void sanitizeEnv(Map environment, Path pwd, YarnConfiguration.DEFAULT_NM_USER_HOME_DIR ) ); - + nmVars.add(Environment.HOME.name()); + environment.put(Environment.PWD.name(), pwd.toString()); - - putEnvIfAbsent(environment, Environment.HADOOP_CONF_DIR.name()); + nmVars.add(Environment.PWD.name()); if (!Shell.WINDOWS) { environment.put("JVM_PID", "$$"); + nmVars.add("JVM_PID"); } // variables here will be forced in, even if the container has specified them. - Apps.setEnvFromInputString(environment, conf.get( - YarnConfiguration.NM_ADMIN_USER_ENV, - YarnConfiguration.DEFAULT_NM_ADMIN_USER_ENV), File.pathSeparator); + String nmAdminUserEnv = conf.get( + YarnConfiguration.NM_ADMIN_USER_ENV, + YarnConfiguration.DEFAULT_NM_ADMIN_USER_ENV); + Apps.setEnvFromInputString(environment, nmAdminUserEnv, File.pathSeparator); + ArrayList envList = Apps.getEnvVarsFromInputString(nmAdminUserEnv, + File.pathSeparator); + nmVars.addAll(envList); // TODO: Remove Windows check and use this approach on all platforms after // additional testing. See YARN-358. if (Shell.WINDOWS) { sanitizeWindowsEnv(environment, pwd, - resources, nmPrivateClasspathJarDir); + resources, nmPrivateClasspathJarDir, nmVars); } // put AuxiliaryService data to environment for (Map.Entry meta : containerManager .getAuxServiceMetaData().entrySet()) { AuxiliaryServiceHelper.setServiceDataIntoEnv( meta.getKey(), meta.getValue(), environment); + nmVars.add(AuxiliaryServiceHelper.getPrefixServiceName(meta.getKey())); } } private void sanitizeWindowsEnv(Map environment, Path pwd, - Map> resources, Path nmPrivateClasspathJarDir) + Map> resources, Path nmPrivateClasspathJarDir, + LinkedHashSet nmVars) throws IOException { String inputClassPath = environment.get(Environment.CLASSPATH.name()); @@ -1563,6 +1597,7 @@ private void sanitizeWindowsEnv(Map environment, Path pwd, new Path(jarCp[0]), pwd, container.getUser()); String replacementClassPath = localizedClassPathJar.toString() + jarCp[1]; environment.put(Environment.CLASSPATH.name(), replacementClassPath); + nmVars.add(Environment.CLASSPATH.name()); } } diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/DefaultLinuxContainerRuntime.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/DefaultLinuxContainerRuntime.java index b50d56c0e01..83380ee2d4b 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/DefaultLinuxContainerRuntime.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/DefaultLinuxContainerRuntime.java @@ -37,7 +37,6 @@ import org.slf4j.LoggerFactory; import java.util.List; -import java.util.Map; import static org.apache.hadoop.yarn.server.nodemanager.containermanager.linux.runtime.LinuxContainerRuntimeConstants.*; @@ -73,11 +72,6 @@ public void initialize(Configuration conf, Context nmContext) this.conf = conf; } - @Override - public boolean useWhitelistEnv(Map env) { - return true; - } - @Override public void prepareContainer(ContainerRuntimeContext ctx) throws ContainerExecutionException { diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/DelegatingLinuxContainerRuntime.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/DelegatingLinuxContainerRuntime.java index dd10617a81c..675bffb00c2 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/DelegatingLinuxContainerRuntime.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/DelegatingLinuxContainerRuntime.java @@ -94,17 +94,6 @@ public void initialize(Configuration conf, Context nmContext) } } - @Override - public boolean useWhitelistEnv(Map env) { - try { - LinuxContainerRuntime runtime = pickContainerRuntime(env); - return runtime.useWhitelistEnv(env); - } catch (ContainerExecutionException e) { - LOG.debug("Unable to determine runtime"); - return false; - } - } - @VisibleForTesting LinuxContainerRuntime pickContainerRuntime( Map environment) throws ContainerExecutionException { diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/DockerLinuxContainerRuntime.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/DockerLinuxContainerRuntime.java index 601c32c205a..26af9a0a47a 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/DockerLinuxContainerRuntime.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/DockerLinuxContainerRuntime.java @@ -366,13 +366,6 @@ public void initialize(Configuration conf, Context nmContext) return capabilities; } - @Override - public boolean useWhitelistEnv(Map env) { - // Avoid propagating nodemanager environment variables into the container - // so those variables can be picked up from the Docker image instead. - return false; - } - private String runDockerVolumeCommand(DockerVolumeCommand dockerVolumeCommand, Container container) throws ContainerExecutionException { try { diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/runtime/ContainerRuntime.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/runtime/ContainerRuntime.java index aa294fc57c2..7caa0edf4de 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/runtime/ContainerRuntime.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/runtime/ContainerRuntime.java @@ -24,8 +24,6 @@ import org.apache.hadoop.classification.InterfaceStability; import org.apache.hadoop.yarn.server.nodemanager.containermanager.container.Container; -import java.util.Map; - /** * An abstraction for various container runtime implementations. Examples * include Process Tree, Docker, Appc runtimes etc. These implementations @@ -85,13 +83,4 @@ void reapContainer(ContainerRuntimeContext ctx) * and hostname */ String[] getIpAndHost(Container container) throws ContainerExecutionException; - - /** - * Whether to propagate the whitelist of environment variables from the - * nodemanager environment into the container environment. - * @param env the container's environment variables - * @return true if whitelist variables should be propagated, false otherwise - * @see org.apache.hadoop.yarn.conf.YarnConfiguration#NM_ENV_WHITELIST - */ - boolean useWhitelistEnv(Map env); } \ No newline at end of file diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/launcher/TestContainerLaunch.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/launcher/TestContainerLaunch.java index 5923f8ef055..81562732597 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/launcher/TestContainerLaunch.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/launcher/TestContainerLaunch.java @@ -41,6 +41,7 @@ import java.util.Collections; import java.util.HashMap; import java.util.LinkedHashMap; +import java.util.LinkedHashSet; import java.util.List; import java.util.Map; import java.util.StringTokenizer; @@ -186,7 +187,7 @@ public void testSpecialCharSymlinks() throws IOException { new DefaultContainerExecutor(); defaultContainerExecutor.setConf(new YarnConfiguration()); defaultContainerExecutor.writeLaunchEnv(fos, env, resources, commands, - new Path(localLogDir.getAbsolutePath()), "user", tempFile.getName()); + new Path(localLogDir.getAbsolutePath()), "user", tempFile.getName(), null); fos.flush(); fos.close(); FileUtil.setExecutable(tempFile, true); @@ -261,7 +262,7 @@ public void testInvalidSymlinkDiagnostics() throws IOException { new DefaultContainerExecutor(); defaultContainerExecutor.setConf(new YarnConfiguration()); defaultContainerExecutor.writeLaunchEnv(fos, env, resources, commands, - new Path(localLogDir.getAbsolutePath()), "user"); + new Path(localLogDir.getAbsolutePath()), "user", null); fos.flush(); fos.close(); FileUtil.setExecutable(tempFile, true); @@ -324,7 +325,7 @@ protected String getNMEnvVar(String varname) { "HADOOP_MAPRED_HOME,HADOOP_YARN_HOME"); defaultContainerExecutor.setConf(conf); defaultContainerExecutor.writeLaunchEnv(fos, env, resources, commands, - new Path(localLogDir.getAbsolutePath()), "user"); + new Path(localLogDir.getAbsolutePath()), "user", null); String shellContent = new String(Files.readAllBytes(Paths.get(shellFile.getAbsolutePath())), StandardCharsets.UTF_8); @@ -337,7 +338,8 @@ protected String getNMEnvVar(String varname) { Assert.assertFalse(shellContent.contains("HADOOP_HDFS_HOME")); // Available in env and in whitelist Assert.assertTrue(shellContent.contains( - "export HADOOP_YARN_HOME=\"nodemanager_yarn_home\"")); + "export HADOOP_YARN_HOME=${HADOOP_YARN_HOME:-\"nodemanager_yarn_home\"}" + )); fos.flush(); fos.close(); } @@ -373,7 +375,7 @@ protected String getNMEnvVar(String varname) { "HADOOP_MAPRED_HOME,HADOOP_YARN_HOME"); lce.setConf(conf); lce.writeLaunchEnv(fos, env, resources, commands, - new Path(localLogDir.getAbsolutePath()), "user"); + new Path(localLogDir.getAbsolutePath()), "user", null); String shellContent = new String(Files.readAllBytes(Paths.get(shellFile.getAbsolutePath())), StandardCharsets.UTF_8); @@ -382,13 +384,106 @@ protected String getNMEnvVar(String varname) { // Whitelisted variable overridden by container Assert.assertTrue(shellContent.contains( "export HADOOP_MAPRED_HOME=\"/opt/hadoopbuild\"")); - // Verify no whitelisted variables inherited from NM env + // Available in env but not in whitelist Assert.assertFalse(shellContent.contains("HADOOP_HDFS_HOME")); - Assert.assertFalse(shellContent.contains("HADOOP_YARN_HOME")); + // Available in env and in whitelist + Assert.assertTrue(shellContent.contains( + "export HADOOP_YARN_HOME=${HADOOP_YARN_HOME:-\"nodemanager_yarn_home\"}" + )); + fos.flush(); + fos.close(); + } + + @Test(timeout = 20000) + public void testWriteEnvOrder() throws Exception { + // Valid only for unix + assumeNotWindows(); + List commands = new ArrayList(); + + // Setup user-defined environment + Map env = new HashMap(); + env.put("USER_VAR_1", "1"); + env.put("USER_VAR_2", "2"); + env.put("NM_MODIFIED_VAR_1", "nm 1"); + env.put("NM_MODIFIED_VAR_2", "nm 2"); + + // These represent vars explicitly set by NM + LinkedHashSet trackedNmVars = new LinkedHashSet<>(); + trackedNmVars.add("NM_MODIFIED_VAR_1"); + trackedNmVars.add("NM_MODIFIED_VAR_2"); + + // Setup Nodemanager environment + final Map nmEnv = new HashMap<>(); + nmEnv.put("WHITELIST_VAR_1", "wl 1"); + nmEnv.put("WHITELIST_VAR_2", "wl 2"); + nmEnv.put("NON_WHITELIST_VAR_1", "nwl 1"); + nmEnv.put("NON_WHITELIST_VAR_2", "nwl 2"); + DefaultContainerExecutor defaultContainerExecutor = + new DefaultContainerExecutor() { + @Override + protected String getNMEnvVar(String varname) { + return nmEnv.get(varname); + } + }; + + // Setup conf with whitelisted variables + ArrayList whitelistVars = new ArrayList<>(); + whitelistVars.add("WHITELIST_VAR_1"); + whitelistVars.add("WHITELIST_VAR_2"); + YarnConfiguration conf = new YarnConfiguration(); + conf.set(YarnConfiguration.NM_ENV_WHITELIST, + whitelistVars.get(0) + "," + whitelistVars.get(1)); + + // These are in the NM env, but not in the whitelist. + ArrayList nonWhiteListEnv = new ArrayList<>(); + nonWhiteListEnv.add("NON_WHITELIST_VAR_1"); + nonWhiteListEnv.add("NON_WHITELIST_VAR_2"); + + // Write the launch script + File shellFile = Shell.appendScriptExtension(tmpDir, "hello"); + Map> resources = new HashMap>(); + FileOutputStream fos = new FileOutputStream(shellFile); + defaultContainerExecutor.setConf(conf); + defaultContainerExecutor.writeLaunchEnv(fos, env, resources, commands, + new Path(localLogDir.getAbsolutePath()), "user", trackedNmVars); fos.flush(); fos.close(); + + // Examine the script + String shellContent = + new String(Files.readAllBytes(Paths.get(shellFile.getAbsolutePath())), + StandardCharsets.UTF_8); + // First make sure everything is there that's supposed to be + for (String envVar : env.keySet()) { + Assert.assertTrue(shellContent.contains(envVar + "=")); + } + for (String wlVar : whitelistVars) { + Assert.assertTrue(shellContent.contains(wlVar + "=")); + } + for (String nwlVar : nonWhiteListEnv) { + Assert.assertFalse(shellContent.contains(nwlVar + "=")); + } + // Explicitly Set NM vars should be before user vars + for (String nmVar : trackedNmVars) { + for (String userVar : env.keySet()) { + // Need to skip nm vars and whitelist vars + if (!trackedNmVars.contains(userVar) && + !whitelistVars.contains(userVar)) { + Assert.assertTrue(shellContent.indexOf(nmVar + "=") < + shellContent.indexOf(userVar + "=")); + } + } + } + // Whitelisted vars should be before explicitly set NM vars + for (String wlVar : whitelistVars) { + for (String nmVar : trackedNmVars) { + Assert.assertTrue(shellContent.indexOf(wlVar + "=") < + shellContent.indexOf(nmVar + "=")); + } + } } + @Test (timeout = 20000) public void testInvalidEnvSyntaxDiagnostics() throws IOException { @@ -411,7 +506,7 @@ public void testInvalidEnvSyntaxDiagnostics() throws IOException { new DefaultContainerExecutor(); defaultContainerExecutor.setConf(new YarnConfiguration()); defaultContainerExecutor.writeLaunchEnv(fos, env, resources, commands, - new Path(localLogDir.getAbsolutePath()), "user"); + new Path(localLogDir.getAbsolutePath()), "user", null); fos.flush(); fos.close(); @@ -494,7 +589,7 @@ public void testContainerLaunchStdoutAndStderrDiagnostics() throws IOException { ContainerExecutor exec = new DefaultContainerExecutor(); exec.setConf(new YarnConfiguration()); exec.writeLaunchEnv(fos, env, resources, commands, - new Path(localLogDir.getAbsolutePath()), "user"); + new Path(localLogDir.getAbsolutePath()), "user", null); fos.flush(); fos.close(); @@ -575,6 +670,7 @@ public void handle(Event event) { List appDirs = new ArrayList(); List userLocalDirs = new ArrayList<>(); List containerLogs = new ArrayList(); + LinkedHashSet nmVars = new LinkedHashSet<>(); Map> resources = new HashMap>(); Path userjar = new Path("user.jar"); @@ -585,7 +681,7 @@ public void handle(Event event) { Path nmp = new Path(testDir); launch.sanitizeEnv(userSetEnv, pwd, appDirs, userLocalDirs, containerLogs, - resources, nmp); + resources, nmp, nmVars); List result = getJarManifestClasspath(userSetEnv.get(Environment.CLASSPATH.name())); @@ -604,7 +700,7 @@ public void handle(Event event) { dispatcher, exec, null, container, dirsHandler, containerManager); launch.sanitizeEnv(userSetEnv, pwd, appDirs, userLocalDirs, containerLogs, - resources, nmp); + resources, nmp, nmVars); result = getJarManifestClasspath(userSetEnv.get(Environment.CLASSPATH.name())); @@ -1530,7 +1626,7 @@ public void testDebuggingInformation() throws IOException { exec.setConf(conf); exec.writeLaunchEnv(fos, env, resources, commands, new Path(localLogDir.getAbsolutePath()), "user", - tempFile.getName()); + tempFile.getName(), null); fos.flush(); fos.close(); FileUtil.setExecutable(tempFile, true); @@ -1754,7 +1850,7 @@ private void validateShellExecutorForDifferentEnvs(Map env) DefaultContainerExecutor executor = new DefaultContainerExecutor(); executor.setConf(new Configuration()); executor.writeLaunchEnv(fos, env, resources, commands, - new Path(localLogDir.getAbsolutePath()), user); + new Path(localLogDir.getAbsolutePath()), user, null); fos.flush(); fos.close(); @@ -1799,7 +1895,7 @@ public void testValidEnvVariableSubstitution() throws IOException { execConf.setBoolean(YarnConfiguration.NM_LOG_CONTAINER_DEBUG_INFO, false); executor.setConf(execConf); executor.writeLaunchEnv(fos, env, resources, commands, - new Path(localLogDir.getAbsolutePath()), user); + new Path(localLogDir.getAbsolutePath()), user, null); fos.flush(); fos.close();