commit d532d2da83bd8489a4a6685b0625419ec65880df Author: Jian He Date: Tue Mar 11 17:56:47 2014 -0700 YARN-1824 diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/ApplicationConstants.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/ApplicationConstants.java index f2e5138..28ae6d4 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/ApplicationConstants.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/ApplicationConstants.java @@ -59,6 +59,24 @@ */ public static final String LOG_DIR_EXPANSION_VAR = ""; + /** + * This constant is used to construct class path and it will be replaced with + * real class path separator(':' for Linux and ';' for Windows) by + * NodeManager on container launch. User has to use this constant to construct + * class path if user wants cross-platform practice i.e. submit an application + * from a Windows client to a Linux/Unix server or vice versa. + */ + public static final String CLASS_PATH_SEPARATOR= ""; + + /** + * The two constants are used to expand parameter and it will be replaced with + * real parameter expansion marker ('%' for Windows and '$' for Linux) by + * NodeManager on container launch. For example: {{VAR}} will be replaced as + * $VAR on Linux, and %VAR% on Windows. + */ + public static final String PARAMETER_EXPANSION_LEFT="{{"; + public static final String PARAMETER_EXPANSION_RIGHT="}}"; + public static final String STDERR = "stderr"; public static final String STDOUT = "stdout"; @@ -214,5 +232,9 @@ public String toString() { return "$" + variable; } } + + public String $$() { + return PARAMETER_EXPANSION_LEFT + variable + PARAMETER_EXPANSION_RIGHT; + } } } diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java index dbb8465..746af40 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java @@ -956,9 +956,31 @@ /** * Default CLASSPATH for YARN applications. A comma-separated list of - * CLASSPATH entries + * CLASSPATH entries. The parameter expansion marker will be replaced with + * real parameter expansion marker ('%' for Windows and '$' for Linux) by + * NodeManager on container launch. For example: {{VAR}} will be replaced as + * $VAR on Linux, and %VAR% on Windows. */ - public static final String[] DEFAULT_YARN_APPLICATION_CLASSPATH = { + public static final String[] DEFAULT_YARN_APPLICATION_CLASSPATH_CROSS_ENV= { + ApplicationConstants.Environment.HADOOP_CONF_DIR.$$(), + ApplicationConstants.Environment.HADOOP_COMMON_HOME.$$() + + "/share/hadoop/common/*", + ApplicationConstants.Environment.HADOOP_COMMON_HOME.$$() + + "/share/hadoop/common/lib/*", + ApplicationConstants.Environment.HADOOP_HDFS_HOME.$$() + + "/share/hadoop/hdfs/*", + ApplicationConstants.Environment.HADOOP_HDFS_HOME.$$() + + "/share/hadoop/hdfs/lib/*", + ApplicationConstants.Environment.HADOOP_YARN_HOME.$$() + + "/share/hadoop/yarn/*", + ApplicationConstants.Environment.HADOOP_YARN_HOME.$$() + + "/share/hadoop/yarn/lib/*" }; + /** + * Default CLASSPATH for YARN applications. A comma-separated list of + * CLASSPATH entries constructed based on the client OS environment expansion + * syntax. + */ + public static final String[] DEFAULT_YARN_APPLICATION_CLASSPATH_CLIENT_ENV = { ApplicationConstants.Environment.HADOOP_CONF_DIR.$(), ApplicationConstants.Environment.HADOOP_COMMON_HOME.$() + "/share/hadoop/common/*", diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-distributedshell/src/main/java/org/apache/hadoop/yarn/applications/distributedshell/ApplicationMaster.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-distributedshell/src/main/java/org/apache/hadoop/yarn/applications/distributedshell/ApplicationMaster.java index 228f184..e2a522a 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-distributedshell/src/main/java/org/apache/hadoop/yarn/applications/distributedshell/ApplicationMaster.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-distributedshell/src/main/java/org/apache/hadoop/yarn/applications/distributedshell/ApplicationMaster.java @@ -47,6 +47,8 @@ import org.apache.hadoop.classification.InterfaceAudience; import org.apache.hadoop.classification.InterfaceStability; import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.FileSystem; +import org.apache.hadoop.fs.Path; import org.apache.hadoop.io.DataOutputBuffer; import org.apache.hadoop.io.IOUtils; import org.apache.hadoop.net.NetUtils; @@ -74,7 +76,6 @@ import org.apache.hadoop.yarn.api.records.LocalResource; import org.apache.hadoop.yarn.api.records.LocalResourceType; import org.apache.hadoop.yarn.api.records.LocalResourceVisibility; -import org.apache.hadoop.yarn.api.records.NMToken; import org.apache.hadoop.yarn.api.records.NodeReport; import org.apache.hadoop.yarn.api.records.Priority; import org.apache.hadoop.yarn.api.records.Resource; @@ -223,8 +224,9 @@ private long shellScriptPathLen = 0; // Hardcoded path to shell script in launch container's local env - private static final String ExecShellStringPath = "ExecShellScript.sh"; - private static final String ExecBatScripStringtPath = "ExecBatScript.bat"; + private static final String ExecShellStringPath = Client.SCRIPT_PATH + ".sh"; + private static final String ExecBatScripStringtPath = Client.SCRIPT_PATH + + ".bat"; // Hardcoded path to custom log_properties private static final String log4jPath = "log4j.properties"; @@ -846,15 +848,28 @@ public void run() { // In this scenario, if a shell script is specified, we need to have it // copied and made available to the container. if (!shellScriptPath.isEmpty()) { + Path renamedSchellScriptPath = null; + if (Shell.WINDOWS) { + renamedSchellScriptPath = new Path(shellScriptPath + ".bat"); + } else { + renamedSchellScriptPath = new Path(shellScriptPath + ".sh"); + } + try { + FileSystem fs = FileSystem.get(conf); + fs.rename(new Path(shellScriptPath), renamedSchellScriptPath); + } catch (IOException e) { + LOG.warn("Not able to add suffix (.bat/.sh) to the shell script filename"); + } + LocalResource shellRsrc = Records.newRecord(LocalResource.class); shellRsrc.setType(LocalResourceType.FILE); shellRsrc.setVisibility(LocalResourceVisibility.APPLICATION); try { shellRsrc.setResource(ConverterUtils.getYarnUrlFromURI(new URI( - shellScriptPath))); + renamedSchellScriptPath.toString()))); } catch (URISyntaxException e) { LOG.error("Error when trying to use shell script path specified" - + " in env, path=" + shellScriptPath); + + " in env, path=" + renamedSchellScriptPath); e.printStackTrace(); // A failure scenario on bad input such as invalid shell script path diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-distributedshell/src/main/java/org/apache/hadoop/yarn/applications/distributedshell/Client.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-distributedshell/src/main/java/org/apache/hadoop/yarn/applications/distributedshell/Client.java index 333486d..e872c92 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-distributedshell/src/main/java/org/apache/hadoop/yarn/applications/distributedshell/Client.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-distributedshell/src/main/java/org/apache/hadoop/yarn/applications/distributedshell/Client.java @@ -18,7 +18,6 @@ package org.apache.hadoop.yarn.applications.distributedshell; -import java.io.File; import java.io.IOException; import java.nio.ByteBuffer; import java.util.ArrayList; @@ -49,7 +48,6 @@ import org.apache.hadoop.security.Credentials; import org.apache.hadoop.security.UserGroupInformation; import org.apache.hadoop.security.token.Token; -import org.apache.hadoop.util.Shell; import org.apache.hadoop.yarn.api.ApplicationClientProtocol; import org.apache.hadoop.yarn.api.ApplicationConstants; import org.apache.hadoop.yarn.api.ApplicationConstants.Environment; @@ -177,8 +175,7 @@ // Hardcoded path to custom log_properties private static final String log4jPath = "log4j.properties"; - private static final String linuxShellPath = "ExecShellScript.sh"; - private static final String windowBatPath = "ExecBatScript.bat"; + public static final String SCRIPT_PATH = "ExecScript"; /** * @param args Command line arguments @@ -492,8 +489,7 @@ public boolean run() throws IOException, YarnException { if (!shellScriptPath.isEmpty()) { Path shellSrc = new Path(shellScriptPath); String shellPathSuffix = - appName + "/" + appId.getId() + "/" - + (Shell.WINDOWS ? windowBatPath : linuxShellPath); + appName + "/" + appId.getId() + "/" + SCRIPT_PATH; Path shellDst = new Path(fs.getHomeDirectory(), shellPathSuffix); fs.copyFromLocalFile(false, true, shellSrc, shellDst); @@ -535,15 +531,16 @@ public boolean run() throws IOException, YarnException { // It should be provided out of the box. // For now setting all required classpaths including // the classpath to "." for the application jar - StringBuilder classPathEnv = new StringBuilder(Environment.CLASSPATH.$()) - .append(File.pathSeparatorChar).append("./*"); + StringBuilder classPathEnv = new StringBuilder(Environment.CLASSPATH.$$()) + .append(ApplicationConstants.CLASS_PATH_SEPARATOR).append("./*"); for (String c : conf.getStrings( YarnConfiguration.YARN_APPLICATION_CLASSPATH, - YarnConfiguration.DEFAULT_YARN_APPLICATION_CLASSPATH)) { - classPathEnv.append(File.pathSeparatorChar); + YarnConfiguration.DEFAULT_YARN_APPLICATION_CLASSPATH_CROSS_ENV)) { + classPathEnv.append(ApplicationConstants.CLASS_PATH_SEPARATOR); classPathEnv.append(c.trim()); } - classPathEnv.append(File.pathSeparatorChar).append("./log4j.properties"); + classPathEnv.append(ApplicationConstants.CLASS_PATH_SEPARATOR).append( + "./log4j.properties"); // add the runtime classpath needed for tests to work if (conf.getBoolean(YarnConfiguration.IS_MINI_YARN_CLUSTER, false)) { @@ -560,7 +557,7 @@ public boolean run() throws IOException, YarnException { // Set java executable command LOG.info("Setting up app master command"); - vargs.add(Environment.JAVA_HOME.$() + "/bin/java"); + vargs.add(Environment.JAVA_HOME.$$() + "/bin/java"); // Set Xmx based on am memory size vargs.add("-Xmx" + amMemory + "m"); // Set class name diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/Apps.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/Apps.java index a357734..82d7253 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/Apps.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/Apps.java @@ -33,6 +33,7 @@ import org.apache.hadoop.classification.InterfaceStability.Unstable; import org.apache.hadoop.util.Shell; import org.apache.hadoop.util.StringInterner; +import org.apache.hadoop.yarn.api.ApplicationConstants; import org.apache.hadoop.yarn.api.records.ApplicationId; import org.apache.hadoop.yarn.exceptions.YarnRuntimeException; @@ -70,7 +71,7 @@ public static void throwParseException(String name, String s) { } public static void setEnvFromInputString(Map env, - String envString) { + String envString, String classPathSeparator) { if (envString != null && envString.length() > 0) { String childEnvs[] = envString.split(","); Pattern p = Pattern.compile(Shell.getEnvironmentVariableRegex()); @@ -92,7 +93,7 @@ public static void setEnvFromInputString(Map env, m.appendReplacement(sb, Matcher.quoteReplacement(replace)); } m.appendTail(sb); - addToEnvironment(env, parts[0], sb.toString()); + addToEnvironment(env, parts[0], sb.toString(), classPathSeparator); } } } @@ -101,14 +102,19 @@ public static void setEnvFromInputString(Map env, @Unstable public static void addToEnvironment( Map environment, - String variable, String value) { + String variable, String value, String classPathSeparator) { String val = environment.get(variable); if (val == null) { val = value; } else { - val = val + File.pathSeparator + value; + val = val + classPathSeparator + value; } environment.put(StringInterner.weakIntern(variable), StringInterner.weakIntern(val)); } + + public static String expandEnvironment(String var) { + return ApplicationConstants.PARAMETER_EXPANSION_LEFT + var + + ApplicationConstants.PARAMETER_EXPANSION_RIGHT; + } } diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/launcher/ContainerLaunch.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/launcher/ContainerLaunch.java index 8b08965..2c87ee0 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/launcher/ContainerLaunch.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/launcher/ContainerLaunch.java @@ -76,6 +76,8 @@ import org.apache.hadoop.yarn.util.AuxiliaryServiceHelper; import org.apache.hadoop.yarn.util.ConverterUtils; +import com.google.common.annotations.VisibleForTesting; + public class ContainerLaunch implements Callable { private static final Log LOG = LogFactory.getLog(ContainerLaunch.class); @@ -124,6 +126,25 @@ public ContainerLaunch(Context context, Configuration configuration, YarnConfiguration.DEFAULT_NM_PROCESS_KILL_WAIT_MS); } + @VisibleForTesting + public static String expandEnvironment(String var, + Path containerLogDir) { + var = var.replace(ApplicationConstants.LOG_DIR_EXPANSION_VAR, + containerLogDir.toString()); + var = var.replace(ApplicationConstants.CLASS_PATH_SEPARATOR, + File.pathSeparator); + + // replace parameter expansion marker. e.g. {{VAR}} on Windows is replaced + // as %VAR% and on Linux replaced as "$VAR" + if (Shell.WINDOWS) { + var = var.replaceAll("(\\{\\{)|(\\}\\})", "%"); + } else { + var = var.replace("{{", "$"); + var = var.replace("}}", ""); + } + return var; + } + @Override @SuppressWarnings("unchecked") // dispatcher not typed public Integer call() { @@ -165,8 +186,7 @@ public Integer call() { dirsHandler.getLogPathForWrite(relativeContainerLogDir, false); for (String str : command) { // TODO: Should we instead work via symlinks without this grammar? - newCmds.add(str.replace(ApplicationConstants.LOG_DIR_EXPANSION_VAR, - containerLogDir.toString())); + newCmds.add(expandEnvironment(str, containerLogDir)); } launchContext.setCommands(newCmds); @@ -174,11 +194,8 @@ public Integer call() { // Make a copy of env to iterate & do variable expansion for (Entry entry : environment.entrySet()) { String value = entry.getValue(); - entry.setValue( - value.replace( - ApplicationConstants.LOG_DIR_EXPANSION_VAR, - containerLogDir.toString()) - ); + value = expandEnvironment(value, containerLogDir); + entry.setValue(value); } // /////////////////////////// End of variable expansion @@ -647,12 +664,9 @@ public void sanitizeEnv(Map environment, Path pwd, } // variables here will be forced in, even if the container has specified them. - Apps.setEnvFromInputString( - environment, - conf.get( - YarnConfiguration.NM_ADMIN_USER_ENV, - YarnConfiguration.DEFAULT_NM_ADMIN_USER_ENV) - ); + Apps.setEnvFromInputString(environment, conf.get( + YarnConfiguration.NM_ADMIN_USER_ENV, + YarnConfiguration.DEFAULT_NM_ADMIN_USER_ENV), File.pathSeparator); // TODO: Remove Windows check and use this approach on all platforms after // additional testing. See YARN-358. diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/launcher/TestContainerLaunch.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/launcher/TestContainerLaunch.java index 81cf797..982f162 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/launcher/TestContainerLaunch.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/launcher/TestContainerLaunch.java @@ -21,7 +21,6 @@ import static org.junit.Assert.assertEquals; import static org.mockito.Mockito.mock; import static org.mockito.Mockito.when; -import static org.mockito.Mockito.spy; import java.io.BufferedReader; import java.io.File; @@ -48,6 +47,7 @@ import org.apache.hadoop.util.Shell; import org.apache.hadoop.util.Shell.ExitCodeException; import org.apache.hadoop.util.StringUtils; +import org.apache.hadoop.yarn.api.ApplicationConstants; import org.apache.hadoop.yarn.api.ApplicationConstants.Environment; import org.apache.hadoop.yarn.api.protocolrecords.GetContainerStatusesRequest; import org.apache.hadoop.yarn.api.protocolrecords.StartContainerRequest; @@ -73,12 +73,12 @@ import org.apache.hadoop.yarn.server.nodemanager.ContainerExecutor.ExitCode; import org.apache.hadoop.yarn.server.nodemanager.DefaultContainerExecutor; import org.apache.hadoop.yarn.server.nodemanager.containermanager.BaseContainerManagerTest; -import org.apache.hadoop.yarn.server.nodemanager.containermanager.ContainerManagerImpl; import org.apache.hadoop.yarn.server.nodemanager.containermanager.container.Container; import org.apache.hadoop.yarn.server.nodemanager.containermanager.container.ContainerEventType; import org.apache.hadoop.yarn.server.nodemanager.containermanager.container.ContainerExitEvent; import org.apache.hadoop.yarn.server.nodemanager.containermanager.localizer.ContainerLocalizer; import org.apache.hadoop.yarn.server.utils.BuilderUtils; +import org.apache.hadoop.yarn.util.Apps; import org.apache.hadoop.yarn.util.AuxiliaryServiceHelper; import org.apache.hadoop.yarn.util.ConverterUtils; import org.apache.hadoop.yarn.util.LinuxResourceCalculatorPlugin; @@ -287,6 +287,31 @@ public void testInvalidEnvSyntaxDiagnostics() throws IOException { } } + @Test(timeout = 10000) + public void testEnvExpansion() throws IOException { + Path logPath = new Path("/nm/container/logs"); + String input = + Apps.expandEnvironment("HADOOP_HOME") + "/share/hadoop/common/*" + + ApplicationConstants.CLASS_PATH_SEPARATOR + + Apps.expandEnvironment("HADOOP_HOME") + "/share/hadoop/common/lib/*" + + ApplicationConstants.CLASS_PATH_SEPARATOR + + Apps.expandEnvironment("HADOOP_LOG_HOME") + + ApplicationConstants.LOG_DIR_EXPANSION_VAR; + + String res = ContainerLaunch.expandEnvironment(input, logPath); + + if (Shell.WINDOWS) { + Assert.assertEquals("%HADOOP_HOME%/share/hadoop/common/*;" + + "%HADOOP_HOME%/share/hadoop/common/lib/*;" + + "%HADOOP_LOG_HOME%/nm/container/logs", res); + } else { + Assert.assertEquals("$HADOOP_HOME/share/hadoop/common/*:" + + "$HADOOP_HOME/share/hadoop/common/lib/*:" + + "$HADOOP_LOG_HOME/nm/container/logs", res); + } + System.out.println(res); + } + @Test (timeout = 20000) public void testContainerLaunchStdoutAndStderrDiagnostics() throws IOException {