diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/Apps.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/Apps.java index 1c90d551b7b..ec670cd919e 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/Apps.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/Apps.java @@ -52,7 +52,13 @@ "(?<=^|,)" // preceded by ',' or line begin + '(' + Shell.ENV_NAME_REGEX + ')' // var group + '=' - + "([^,]*)" // val group + + "(" // val group + + ".{0}" + "|" // empty value + + "'[^']*'" + "|" // single quoted string + + "\"[^\"]*\"" + "|" // double quoted string + + "[^,]+" // non-commas + + ")" + + "(?=,|$)" ); public static ApplicationId toAppID(String aid) { diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/util/TestApps.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/util/TestApps.java index 66d2d231344..5a74bbfc20c 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/util/TestApps.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/util/TestApps.java @@ -31,25 +31,69 @@ public void testSetEnvFromInputString() { Map environment = new HashMap(); environment.put("JAVA_HOME", "/path/jdk"); - String goodEnv = "a1=1,b_2=2,_c=3,d=4,e=,f_win=%JAVA_HOME%" - + ",g_nix=$JAVA_HOME"; + String goodEnv = + "a1=1" + + ",b_2=2" + + ",_c=3" + + ",d=4" + + ",e=" + + ",f_win=%JAVA_HOME%" + + ",g_nix=$JAVA_HOME" + + ",h='6,7'" + + ",i=\"8,9\"" + + ",j=\"abc'def','ghi',jkl\"" + + ",k='string with \"quotes\", $JAVA_HOME, and %JAVA_HOME%'" + + ",l=''" + ",m=\"\"" + + ",n='single quotes'-and-\"double quotes\"" + + ",o=\"double quotes\"-and-'single quotes'" + + ",p1=\"quoted string with \"quote\" in it\"" + + ",p2='quoted string with 'quote' in it'" + + ",q=\"quoted,string,with,commas,\"" + + ",r=',,,,,,,'" + + ",s=stringwith'single'\"double\"init" + + ",YARN_DOCKER_CONTAINER_RUNTIME_MOUNTS='/a/b/c:/d/e/f:ro,/g:/h/g:ro'" + ; Apps.setEnvFromInputString(environment, goodEnv, File.pathSeparator); assertEquals("1", environment.get("a1")); assertEquals("2", environment.get("b_2")); assertEquals("3", environment.get("_c")); assertEquals("4", environment.get("d")); assertEquals("", environment.get("e")); + assertEquals("'6,7'", environment.get("h")); + assertEquals("\"8,9\"", environment.get("i")); + assertEquals("\"abc'def','ghi',jkl\"", environment.get("j")); + assertEquals("''", environment.get("l")); + assertEquals("\"\"", environment.get("m")); + assertEquals("'single quotes'-and-\"double quotes\"", + environment.get("n")); + assertEquals("\"double quotes\"-and-'single quotes'", + environment.get("o")); + assertEquals("\"quoted string with \"quote\" in it\"", + environment.get("p1")); + assertEquals("'quoted string with 'quote' in it'", + environment.get("p2")); + assertEquals("\"quoted,string,with,commas,\"", + environment.get("q")); + assertEquals("',,,,,,,'", environment.get("r")); + assertEquals("stringwith'single'\"double\"init", + environment.get("s")); + assertEquals("'/a/b/c:/d/e/f:ro,/g:/h/g:ro'", + environment.get("YARN_DOCKER_CONTAINER_RUNTIME_MOUNTS")); if (Shell.WINDOWS) { assertEquals("$JAVA_HOME", environment.get("g_nix")); assertEquals("/path/jdk", environment.get("f_win")); + assertEquals("'string with \"quotes\", $JAVA_HOME, and /path/jdk'", + environment.get("k")); } else { assertEquals("/path/jdk", environment.get("g_nix")); assertEquals("%JAVA_HOME%", environment.get("f_win")); + assertEquals("'string with \"quotes\", /path/jdk, and %JAVA_HOME%'", + environment.get("k")); } String badEnv = "1,,2=a=b,3=a=,4==,5==a,==,c-3=3,="; environment.clear(); Apps.setEnvFromInputString(environment, badEnv, File.pathSeparator); - assertEquals(environment.size(), 0); + assertEquals(0, environment.size()); // Test "=" in the value part environment.clear(); @@ -57,5 +101,22 @@ public void testSetEnvFromInputString() { File.pathSeparator); assertEquals("=", environment.get("e1")); assertEquals("a1=a2", environment.get("e2")); + + // Test empty string + environment.clear(); + Apps.setEnvFromInputString(environment, "", File.pathSeparator); + assertEquals(0, environment.size()); + + // Test quoted string with comma and same quotes + environment.clear(); + Apps.setEnvFromInputString(environment, + "f=\"quoted,string,with,commas,and\"quotes\"\"", File.pathSeparator); + assertEquals("\"quoted", environment.get("f")); + + // Test quoted string with comma and same quotes + environment.clear(); + Apps.setEnvFromInputString(environment, + "g='quoted,string,with,commas,and 'quotes''", File.pathSeparator); + assertEquals("'quoted", environment.get("g")); } } diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/DockerLinuxContainerRuntime.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/DockerLinuxContainerRuntime.java index de225e6a2b1..f5b365fd98b 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/DockerLinuxContainerRuntime.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/DockerLinuxContainerRuntime.java @@ -716,6 +716,20 @@ private String getUserIdInfo(String userName) return id; } + private static String trimEnclosingQuotes(String s) { + // Strip surrounding single or double quotes in s + String trimmedString = s; + if (s.length() >= 2 ) { + char startChar = s.charAt(0); + char endChar = s.charAt(s.length() - 1); + if (((startChar == '"') || (startChar == '\'')) && + (startChar == endChar)) { + trimmedString = s.substring(1, s.length() - 1); + } + } + return trimmedString; + } + @Override public void launchContainer(ContainerRuntimeContext ctx) throws ContainerExecutionException { @@ -811,8 +825,9 @@ public void launchContainer(ContainerRuntimeContext ctx) } if (environment.containsKey(ENV_DOCKER_CONTAINER_MOUNTS)) { - Matcher parsedMounts = USER_MOUNT_PATTERN.matcher( - environment.get(ENV_DOCKER_CONTAINER_MOUNTS)); + String trimmedMounts = + trimEnclosingQuotes(environment.get(ENV_DOCKER_CONTAINER_MOUNTS)); + Matcher parsedMounts = USER_MOUNT_PATTERN.matcher(trimmedMounts); if (!parsedMounts.find()) { throw new ContainerExecutionException( "Unable to parse user supplied mount list: "