diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/ApplicationConstants.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/ApplicationConstants.java index eb03fb2a957..2a083a762bb 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/ApplicationConstants.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/ApplicationConstants.java @@ -97,7 +97,7 @@ /** * The following two constants are used to expand parameter and it will be * replaced with real parameter expansion marker ('%' for Windows and '$' for - * Linux) by NodeManager on container launch. For example: {{VAR}} will be + * Linux) by NodeManager on container launch. For example: {_{VAR}_} will be * replaced as $VAR on Linux, and %VAR% on Windows. User has to use this * constant to construct class path if user wants cross-platform practice i.e. * submit an application from a Windows client to a Linux/Unix server or vice @@ -105,7 +105,7 @@ */ @Public @Unstable - public static final String PARAMETER_EXPANSION_LEFT="{{"; + String PARAMETER_EXPANSION_LEFT="{_{"; /** * User has to use this constant to construct class path if user wants @@ -114,7 +114,7 @@ */ @Public @Unstable - public static final String PARAMETER_EXPANSION_RIGHT="}}"; + String PARAMETER_EXPANSION_RIGHT="}_}"; public static final String STDERR = "stderr"; @@ -314,9 +314,10 @@ public String toString() { /** * Expand the environment variable in platform-agnostic syntax. The - * parameter expansion marker "{{VAR}}" will be replaced with real parameter - * expansion marker ('%' for Windows and '$' for Linux) by NodeManager on - * container launch. For example: {{VAR}} will be replaced as $VAR on Linux, + * parameter expansion marker "{_{VAR}_}" will be replaced + * with real parameter expansion marker ('%' for Windows and + * '$' for Linux) by NodeManager on container launch. + * For example: {_{VAR}_} will be replaced as $VAR on Linux, * and %VAR% on Windows. * @return expanded environment variable. */ diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java index ce38d2762b2..8d5d3e586d8 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java @@ -2384,7 +2384,7 @@ public static boolean isAclEnabled(Configuration conf) { * Default platform-agnostic CLASSPATH for YARN applications. A * comma-separated list of CLASSPATH entries. The parameter expansion marker * will be replaced with real parameter expansion marker ('%' for Windows and - * '$' for Linux) by NodeManager on container launch. For example: {{VAR}} + * '$' for Linux) by NodeManager on container launch. For example: {_{VAR}_} * will be replaced as $VAR on Linux, and %VAR% on Windows. */ @Public diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/launcher/ContainerLaunch.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/launcher/ContainerLaunch.java index 008c09bed61..9faa0a6f018 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/launcher/ContainerLaunch.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/launcher/ContainerLaunch.java @@ -166,10 +166,10 @@ public static String expandEnvironment(String var, var = var.replace(ApplicationConstants.CLASS_PATH_SEPARATOR, File.pathSeparator); - // replace parameter expansion marker. e.g. {{VAR}} on Windows is replaced + // replace parameter expansion marker. e.g. {_{VAR}_} on Windows is replaced // as %VAR% and on Linux replaced as "$VAR" if (Shell.WINDOWS) { - var = var.replaceAll("(\\{\\{)|(\\}\\})", "%"); + var = var.replaceAll("(\\{\\_\\{)|(\\}\\_\\})", "%"); } else { var = var.replace(ApplicationConstants.PARAMETER_EXPANSION_LEFT, "$"); var = var.replace(ApplicationConstants.PARAMETER_EXPANSION_RIGHT, ""); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/launcher/TestContainerLaunch.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/launcher/TestContainerLaunch.java index 1f218d0942c..6d889b97c51 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/launcher/TestContainerLaunch.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/launcher/TestContainerLaunch.java @@ -582,6 +582,25 @@ public void testEnvExpansion() throws IOException { + "$HADOOP_LOG_HOME/nm/container/logs", res); } System.out.println(res); + + /** + * The user commands may contains JSON format string + * which shouldn't be changed. + * */ + input = "echo {\"0\": {\"person\": {\"name\": \"John\"}}}" + + Apps.crossPlatformify("HADOOP_LOG_HOME") + + ApplicationConstants.LOG_DIR_EXPANSION_VAR; + res = ContainerLaunch.expandEnvironment(input, logPath); + + if (Shell.WINDOWS) { + Assert.assertEquals( + "echo {\"0\": {\"person\": {\"name\": \"John\"}}}" + + "%HADOOP_LOG_HOME%/nm/container/logs", res); + } else { + Assert.assertEquals( + "echo {\"0\": {\"person\": {\"name\": \"John\"}}}" + + "$HADOOP_LOG_HOME/nm/container/logs", res); + } } @Test (timeout = 20000)