diff --git hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/StringUtils.java hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/StringUtils.java index fc4b0ab6bf8..8496e1b97ed 100644 --- hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/StringUtils.java +++ hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/StringUtils.java @@ -872,6 +872,11 @@ public static String join(CharSequence separator, Iterable strings) { return sb.toString(); } + public static String join(char separator, Iterable strings) { + return join(separator + "", strings); + } + + /** * Concatenates strings, using a separator. * @@ -894,6 +899,10 @@ public static String join(CharSequence separator, String[] strings) { return sb.toString(); } + public static String join(char separator, String[] strings) { + return join(separator + "", strings); + } + /** * Convert SOME_STUFF to SomeStuff * diff --git hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/util/TestStringUtils.java hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/util/TestStringUtils.java index 515c3e0c0ee..ed92e87d4f7 100644 --- hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/util/TestStringUtils.java +++ hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/util/TestStringUtils.java @@ -277,8 +277,12 @@ public void testJoin() { s.add("c"); assertEquals("", StringUtils.join(":", s.subList(0, 0))); assertEquals("a", StringUtils.join(":", s.subList(0, 1))); + assertEquals("", StringUtils.join(':', s.subList(0, 0))); + assertEquals("a", StringUtils.join(':', s.subList(0, 1))); assertEquals("a:b", StringUtils.join(":", s.subList(0, 2))); assertEquals("a:b:c", StringUtils.join(":", s.subList(0, 3))); + assertEquals("a:b", StringUtils.join(':', s.subList(0, 2))); + assertEquals("a:b:c", StringUtils.join(':', s.subList(0, 3))); } @Test (timeout = 30000) diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/LinuxContainerExecutor.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/LinuxContainerExecutor.java index e4288fe380e..0018698db0a 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/LinuxContainerExecutor.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/LinuxContainerExecutor.java @@ -53,6 +53,8 @@ private static final Log LOG = LogFactory .getLog(LinuxContainerExecutor.class); + public final static char LINUX_FILE_PATH_SEPARATOR = '%'; + private String nonsecureLocalUser; private Pattern nonsecureLocalUserPattern; private String containerExecutorExe; @@ -214,8 +216,10 @@ public void startLocalizer(Path nmPrivateContainerTokensPath, Integer.toString(Commands.INITIALIZE_CONTAINER.getValue()), appId, nmPrivateContainerTokensPath.toUri().getPath().toString(), - StringUtils.join(",", localDirs), - StringUtils.join(",", logDirs))); + StringUtils.join(LINUX_FILE_PATH_SEPARATOR, + localDirs), + StringUtils.join(LINUX_FILE_PATH_SEPARATOR, + logDirs))); File jvm = // use same jvm as parent new File(new File(System.getProperty("java.home"), "bin"), "java"); @@ -286,8 +290,10 @@ public int launchContainer(Container container, nmPrivateCotainerScriptPath.toUri().getPath().toString(), nmPrivateTokensPath.toUri().getPath().toString(), pidFilePath.toString(), - StringUtils.join(",", localDirs), - StringUtils.join(",", logDirs), + StringUtils.join(LINUX_FILE_PATH_SEPARATOR, + localDirs), + StringUtils.join(LINUX_FILE_PATH_SEPARATOR, + logDirs), resourcesOptions)); String[] commandArray = command.toArray(new String[command.size()]); shExec = new ShellCommandExecutor(commandArray, null, // NM's cwd diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/util/CgroupsLCEResourcesHandler.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/util/CgroupsLCEResourcesHandler.java index ffa17ac173d..b45185402d1 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/util/CgroupsLCEResourcesHandler.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/util/CgroupsLCEResourcesHandler.java @@ -409,10 +409,11 @@ public String getResourcesOption(ContainerId containerId) { if (isCpuWeightEnabled()) { sb.append(pathForCgroup(CONTROLLER_CPU, containerName) + "/tasks"); - sb.append(","); + sb.append(LinuxContainerExecutor.LINUX_FILE_PATH_SEPARATOR); } - if (sb.charAt(sb.length() - 1) == ',') { + if (sb.charAt(sb.length() - 1) == + LinuxContainerExecutor.LINUX_FILE_PATH_SEPARATOR) { sb.deleteCharAt(sb.length() - 1); } diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/configuration.c hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/configuration.c index 1667b0d617e..41ea2681e81 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/configuration.c +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/configuration.c @@ -286,14 +286,14 @@ char ** extract_values_delim(char *value, const char *delim) { //first allocate any array of 10 if(value != NULL) { toPass = (char **) malloc(sizeof(char *) * toPassSize); - tempTok = strtok_r((char *)value, ",", &tempstr); + tempTok = strtok_r((char *)value, "%", &tempstr); while (tempTok != NULL) { toPass[size++] = tempTok; if(size == toPassSize) { toPassSize += MAX_SIZE; toPass = (char **) realloc(toPass,(sizeof(char *) * toPassSize)); } - tempTok = strtok_r(NULL, ",", &tempstr); + tempTok = strtok_r(NULL, "%", &tempstr); } } if (toPass != NULL) { @@ -303,7 +303,7 @@ char ** extract_values_delim(char *value, const char *delim) { } /** - * Extracts array of values from the comma separated list of values. + * Extracts array of values from the '%' separated list of values. */ char ** extract_values(char *value) { return extract_values_delim(value, ","); @@ -311,7 +311,7 @@ char ** extract_values(char *value) { /** * Function to return an array of values for a key. - * Value delimiter is assumed to be a comma. + * Value delimiter is assumed to be a '%'. */ char ** get_values(const char * key, struct configuration *cfg) { char *value = get_value(key, cfg); diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/test/test-container-executor.c hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/test/test-container-executor.c index 2375bf4f883..4102779357c 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/test/test-container-executor.c +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/test/test-container-executor.c @@ -42,8 +42,8 @@ #define TEST_ROOT TMPDIR "/test-container-executor" #define DONT_TOUCH_FILE "dont-touch-me" -#define NM_LOCAL_DIRS TEST_ROOT "/local-1," TEST_ROOT "/local-2," \ - TEST_ROOT "/local-3," TEST_ROOT "/local-4," TEST_ROOT "/local-5" +#define NM_LOCAL_DIRS TEST_ROOT "/local-1%" TEST_ROOT "/local-2%" \ + TEST_ROOT "/local-3%" TEST_ROOT "/local-4%" TEST_ROOT "/local-5" #define NM_LOG_DIRS TEST_ROOT "/logs/userlogs" #define ARRAY_SIZE 1000 diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestLinuxContainerExecutorWithMocks.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestLinuxContainerExecutorWithMocks.java index 81591cafa5c..1755d8c5504 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestLinuxContainerExecutorWithMocks.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestLinuxContainerExecutorWithMocks.java @@ -137,8 +137,10 @@ public void testContainerLaunch() throws IOException { assertEquals(Arrays.asList(YarnConfiguration.DEFAULT_NM_NONSECURE_MODE_LOCAL_USER, appSubmitter, cmd, appId, containerId, workDir.toString(), "/bin/echo", "/dev/null", pidFile.toString(), - StringUtils.join(",", dirsHandler.getLocalDirs()), - StringUtils.join(",", dirsHandler.getLogDirs()), "cgroups=none"), + StringUtils.join(LinuxContainerExecutor.LINUX_FILE_PATH_SEPARATOR, + dirsHandler.getLocalDirs()), + StringUtils.join(LinuxContainerExecutor.LINUX_FILE_PATH_SEPARATOR, + dirsHandler.getLogDirs()), "cgroups=none"), readMockParams()); } @@ -283,8 +285,10 @@ public Object answer(InvocationOnMock invocationOnMock) assertEquals(Arrays.asList(YarnConfiguration.DEFAULT_NM_NONSECURE_MODE_LOCAL_USER, appSubmitter, cmd, appId, containerId, workDir.toString(), "/bin/echo", "/dev/null", pidFile.toString(), - StringUtils.join(",", dirsHandler.getLocalDirs()), - StringUtils.join(",", dirsHandler.getLogDirs()), + StringUtils.join(LinuxContainerExecutor.LINUX_FILE_PATH_SEPARATOR, + dirsHandler.getLocalDirs()), + StringUtils.join(LinuxContainerExecutor.LINUX_FILE_PATH_SEPARATOR, + dirsHandler.getLogDirs()), "cgroups=none"), readMockParams()); }