diff --git hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/StringUtils.java hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/StringUtils.java index fc4b0ab..73f9c4f 100644 --- hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/StringUtils.java +++ hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/StringUtils.java @@ -872,6 +872,10 @@ public static String join(CharSequence separator, Iterable strings) { return sb.toString(); } + public static String join(char separator, Iterable strings) { + return join(separator + "", strings); + } + /** * Concatenates strings, using a separator. * @@ -894,6 +898,10 @@ public static String join(CharSequence separator, String[] strings) { return sb.toString(); } + public static String join(char separator, String[] strings) { + return join(separator + "", strings); + } + /** * Convert SOME_STUFF to SomeStuff * diff --git hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/util/TestStringUtils.java hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/util/TestStringUtils.java index 85ab8c4..e3e613c 100644 --- hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/util/TestStringUtils.java +++ hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/util/TestStringUtils.java @@ -278,8 +278,12 @@ public void testJoin() { s.add("c"); assertEquals("", StringUtils.join(":", s.subList(0, 0))); assertEquals("a", StringUtils.join(":", s.subList(0, 1))); + assertEquals("", StringUtils.join(':', s.subList(0, 0))); + assertEquals("a", StringUtils.join(':', s.subList(0, 1))); assertEquals("a:b", StringUtils.join(":", s.subList(0, 2))); assertEquals("a:b:c", StringUtils.join(":", s.subList(0, 3))); + assertEquals("a:b", StringUtils.join(':', s.subList(0, 2))); + assertEquals("a:b:c", StringUtils.join(':', s.subList(0, 3))); } @Test (timeout = 30000) diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/LinuxContainerExecutor.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/LinuxContainerExecutor.java index dbe257d..c7c2cae 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/LinuxContainerExecutor.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/LinuxContainerExecutor.java @@ -74,6 +74,8 @@ private boolean containerLimitUsers; private ResourceHandler resourceHandlerChain; + public final static char LINUX_FILE_PATH_SEPARATOR = '%'; + @Override public void setConf(Configuration conf) { super.setConf(conf); @@ -241,8 +243,8 @@ public void startLocalizer(LocalizerStartContext ctx) Integer.toString(Commands.INITIALIZE_CONTAINER.getValue()), appId, nmPrivateContainerTokensPath.toUri().getPath().toString(), - StringUtils.join(",", localDirs), - StringUtils.join(",", logDirs))); + StringUtils.join(LINUX_FILE_PATH_SEPARATOR, localDirs), + StringUtils.join(LINUX_FILE_PATH_SEPARATOR, logDirs))); File jvm = // use same jvm as parent new File(new File(System.getProperty("java.home"), "bin"), "java"); @@ -363,8 +365,8 @@ public int launchContainer(ContainerStartContext ctx) throws IOException { nmPrivateContainerScriptPath.toUri().getPath().toString(), nmPrivateTokensPath.toUri().getPath().toString(), pidFilePath.toString(), - StringUtils.join(",", localDirs), - StringUtils.join(",", logDirs), + StringUtils.join(LINUX_FILE_PATH_SEPARATOR, localDirs), + StringUtils.join(LINUX_FILE_PATH_SEPARATOR, logDirs), resourcesOptions)); if (tcCommandFile != null) { diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/privileged/PrivilegedOperationExecutor.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/privileged/PrivilegedOperationExecutor.java index 1c4a51c..4e8a86e 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/privileged/PrivilegedOperationExecutor.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/privileged/PrivilegedOperationExecutor.java @@ -30,6 +30,7 @@ import org.apache.hadoop.util.Shell.ExitCodeException; import org.apache.hadoop.yarn.api.ApplicationConstants; import org.apache.hadoop.yarn.conf.YarnConfiguration; +import org.apache.hadoop.yarn.server.nodemanager.LinuxContainerExecutor; import java.io.File; import java.io.IOException; @@ -234,7 +235,7 @@ public String executePrivilegedOperation(PrivilegedOperation operation, if (noneArgsOnly == false) { //We have already appended at least one tasks file. - finalOpArg.append(","); + finalOpArg.append(LinuxContainerExecutor.LINUX_FILE_PATH_SEPARATOR); finalOpArg.append(tasksFile); } else { finalOpArg.append(tasksFile); diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/util/CgroupsLCEResourcesHandler.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/util/CgroupsLCEResourcesHandler.java index b38e559..6a52806 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/util/CgroupsLCEResourcesHandler.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/util/CgroupsLCEResourcesHandler.java @@ -409,10 +409,11 @@ public String getResourcesOption(ContainerId containerId) { if (isCpuWeightEnabled()) { sb.append(pathForCgroup(CONTROLLER_CPU, containerName) + "/tasks"); - sb.append(","); + sb.append(LinuxContainerExecutor.LINUX_FILE_PATH_SEPARATOR); } - if (sb.charAt(sb.length() - 1) == ',') { + if (sb.charAt(sb.length() - 1) == + LinuxContainerExecutor.LINUX_FILE_PATH_SEPARATOR) { sb.deleteCharAt(sb.length() - 1); } diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/configuration.c hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/configuration.c index 15b53ae..51adc97 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/configuration.c +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/configuration.c @@ -283,7 +283,7 @@ char * get_value(const char* key) { /** * Function to return an array of values for a key. - * Value delimiter is assumed to be a comma. + * Value delimiter is assumed to be a '%'. */ char ** get_values(const char * key) { char *value = get_value(key); @@ -291,7 +291,7 @@ char ** get_values(const char * key) { } /** - * Extracts array of values from the comma separated list of values. + * Extracts array of values from the '%' separated list of values. */ char ** extract_values(char *value) { char ** toPass = NULL; @@ -303,14 +303,14 @@ char ** extract_values(char *value) { //first allocate any array of 10 if(value != NULL) { toPass = (char **) malloc(sizeof(char *) * toPassSize); - tempTok = strtok_r((char *)value, ",", &tempstr); + tempTok = strtok_r((char *)value, "%", &tempstr); while (tempTok != NULL) { toPass[size++] = tempTok; if(size == toPassSize) { toPassSize += MAX_SIZE; toPass = (char **) realloc(toPass,(sizeof(char *) * toPassSize)); } - tempTok = strtok_r(NULL, ",", &tempstr); + tempTok = strtok_r(NULL, "%", &tempstr); } } if (toPass != NULL) { diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/test/test-container-executor.c hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/test/test-container-executor.c index be6cc49..13604f0 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/test/test-container-executor.c +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/test/test-container-executor.c @@ -30,8 +30,8 @@ #define TEST_ROOT "/tmp/test-container-executor" #define DONT_TOUCH_FILE "dont-touch-me" -#define NM_LOCAL_DIRS TEST_ROOT "/local-1," TEST_ROOT "/local-2," \ - TEST_ROOT "/local-3," TEST_ROOT "/local-4," TEST_ROOT "/local-5" +#define NM_LOCAL_DIRS TEST_ROOT "/local-1%" TEST_ROOT "/local-2%" \ + TEST_ROOT "/local-3%" TEST_ROOT "/local-4%" TEST_ROOT "/local-5" #define NM_LOG_DIRS TEST_ROOT "/logs/userlogs" #define ARRAY_SIZE 1000 diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestLinuxContainerExecutorWithMocks.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestLinuxContainerExecutorWithMocks.java index d48ce13..884463b 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestLinuxContainerExecutorWithMocks.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestLinuxContainerExecutorWithMocks.java @@ -148,8 +148,10 @@ public void testContainerLaunch() throws IOException { assertEquals(Arrays.asList(YarnConfiguration.DEFAULT_NM_NONSECURE_MODE_LOCAL_USER, appSubmitter, cmd, appId, containerId, workDir.toString(), "/bin/echo", "/dev/null", pidFile.toString(), - StringUtils.join(",", dirsHandler.getLocalDirs()), - StringUtils.join(",", dirsHandler.getLogDirs()), "cgroups=none"), + StringUtils.join(LinuxContainerExecutor.LINUX_FILE_PATH_SEPARATOR, + dirsHandler.getLocalDirs()), + StringUtils.join(LinuxContainerExecutor.LINUX_FILE_PATH_SEPARATOR, + dirsHandler.getLogDirs()), "cgroups=none"), readMockParams()); } @@ -312,8 +314,10 @@ public Object answer(InvocationOnMock invocationOnMock) assertEquals(Arrays.asList(YarnConfiguration.DEFAULT_NM_NONSECURE_MODE_LOCAL_USER, appSubmitter, cmd, appId, containerId, workDir.toString(), "/bin/echo", "/dev/null", pidFile.toString(), - StringUtils.join(",", dirsHandler.getLocalDirs()), - StringUtils.join(",", dirsHandler.getLogDirs()), + StringUtils.join(LinuxContainerExecutor.LINUX_FILE_PATH_SEPARATOR, + dirsHandler.getLocalDirs()), + StringUtils.join(LinuxContainerExecutor.LINUX_FILE_PATH_SEPARATOR, + dirsHandler.getLogDirs()), "cgroups=none"), readMockParams()); }