diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/ApplicationConstants.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/ApplicationConstants.java index d42cd67..3df613c 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/ApplicationConstants.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/ApplicationConstants.java @@ -72,6 +72,17 @@ public static final String CLASS_PATH_SEPARATOR= ""; /** + * This constant is used to construct file path and it will be replaced with + * file path separator('/' for Linux and '\' for Windows) by + * NodeManager on container launch. User has to use this constant to construct + * class path if user wants cross-platform practice i.e. submit an application + * from a Windows client to a Linux/Unix server or vice versa. + */ + @Public + @Unstable + public static final String FILE_PATH_SEPARATOR= ""; + + /** * The following two constants are used to expand parameter and it will be * replaced with real parameter expansion marker ('%' for Windows and '$' for * Linux) by NodeManager on container launch. For example: {{VAR}} will be diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/launcher/ContainerLaunch.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/launcher/ContainerLaunch.java index a87238d..35773de 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/launcher/ContainerLaunch.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/launcher/ContainerLaunch.java @@ -134,7 +134,9 @@ public static String expandEnvironment(String var, var = var.replace(ApplicationConstants.LOG_DIR_EXPANSION_VAR, containerLogDir.toString()); var = var.replace(ApplicationConstants.CLASS_PATH_SEPARATOR, - File.pathSeparator); + File.pathSeparator); + var = var.replace(ApplicationConstants.FILE_PATH_SEPARATOR, + File.separator); // replace parameter expansion marker. e.g. {{VAR}} on Windows is replaced // as %VAR% and on Linux replaced as "$VAR" diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/launcher/TestContainerLaunch.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/launcher/TestContainerLaunch.java index 4088c2a..9cddf9e 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/launcher/TestContainerLaunch.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/launcher/TestContainerLaunch.java @@ -299,10 +299,19 @@ public void testInvalidEnvSyntaxDiagnostics() throws IOException { @Test(timeout = 10000) public void testEnvExpansion() throws IOException { Path logPath = new Path("/nm/container/logs"); + String commonDirPath = ApplicationConstants.FILE_PATH_SEPARATOR + "share" + + ApplicationConstants.FILE_PATH_SEPARATOR + "hadoop" + + ApplicationConstants.FILE_PATH_SEPARATOR + "common"; + String commonPath = + commonDirPath + ApplicationConstants.FILE_PATH_SEPARATOR+ "*"; + String libPath = + commonDirPath + ApplicationConstants.FILE_PATH_SEPARATOR + + "lib"+ ApplicationConstants.FILE_PATH_SEPARATOR + "*"; + String input = - Apps.crossPlatformify("HADOOP_HOME") + "/share/hadoop/common/*" + Apps.crossPlatformify("HADOOP_HOME") + commonPath + ApplicationConstants.CLASS_PATH_SEPARATOR - + Apps.crossPlatformify("HADOOP_HOME") + "/share/hadoop/common/lib/*" + + Apps.crossPlatformify("HADOOP_HOME") + libPath + ApplicationConstants.CLASS_PATH_SEPARATOR + Apps.crossPlatformify("HADOOP_LOG_HOME") + ApplicationConstants.LOG_DIR_EXPANSION_VAR; @@ -310,9 +319,9 @@ public void testEnvExpansion() throws IOException { String res = ContainerLaunch.expandEnvironment(input, logPath); if (Shell.WINDOWS) { - Assert.assertEquals("%HADOOP_HOME%/share/hadoop/common/*;" - + "%HADOOP_HOME%/share/hadoop/common/lib/*;" - + "%HADOOP_LOG_HOME%/nm/container/logs", res); + Assert.assertEquals("%HADOOP_HOME%\\share\\hadoop\\common\\*;" + + "%HADOOP_HOME%\\share/hadoop\\common\\lib\\*;" + + "%HADOOP_LOG_HOME%\\nm\\container\\logs", res); } else { Assert.assertEquals("$HADOOP_HOME/share/hadoop/common/*:" + "$HADOOP_HOME/share/hadoop/common/lib/*:"