commit 671b9c9ea3436e2cc66835a9f12657e1cedb39b6 Author: Eric Yang Date: Mon Feb 25 17:44:42 2019 -0500 YARN-8805. Add ability to use space as delimiter in launch command. Contributed by Eric Yang diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/provider/ProviderUtils.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/provider/ProviderUtils.java index 88883f7..ea1fb0c 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/provider/ProviderUtils.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/provider/ProviderUtils.java @@ -47,9 +47,12 @@ import java.io.FileNotFoundException; import java.io.IOException; import java.io.OutputStream; +import java.util.ArrayList; import java.util.HashMap; +import java.util.List; import java.util.Map; import java.util.concurrent.ExecutionException; +import java.util.regex.Matcher; import java.util.regex.Pattern; import static org.apache.hadoop.yarn.service.api.ServiceApiConstants.COMPONENT_ID; @@ -149,6 +152,20 @@ public static String substituteStrWithTokens(String content, return content; } + public static String replaceSpacesWithDelimiter(String content, + String delimiter) { + List parts = new ArrayList(); + Matcher m = Pattern.compile("([^\"]\\S*|\".+?\")\\s*").matcher(content); + while (m.find()) { + String part = m.group(1); + if(part.startsWith("\"") && part.endsWith("\"")) { + part = part.replaceAll("^\"|\"$", ""); + } + parts.add(part); + } + return String.join(delimiter, parts); + } + // configs will be substituted by corresponding env in tokenMap public static void substituteMapWithTokens(Map configs, Map tokenMap) { diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/provider/docker/DockerProviderService.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/provider/docker/DockerProviderService.java index 6027a66..9b4138e 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/provider/docker/DockerProviderService.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/provider/docker/DockerProviderService.java @@ -25,6 +25,7 @@ import org.apache.hadoop.yarn.api.records.Container; import org.apache.hadoop.yarn.service.api.records.Service; import org.apache.hadoop.yarn.service.utils.SliderFileSystem; + import org.apache.hadoop.yarn.service.containerlaunch.AbstractLauncher; import org.apache.hadoop.yarn.service.containerlaunch.CommandLineBuilder; import org.apache.hadoop.yarn.service.containerlaunch.ContainerLaunchService; @@ -84,6 +85,11 @@ public void buildContainerLaunchCommand(AbstractLauncher launcher, if (useEntryPoint) { String launchCommand = compLaunchContext.getLaunchCommand(); if (!StringUtils.isEmpty(launchCommand)) { + if(launchCommand.contains(" ")) { + // convert space delimiter command to exec format + launchCommand = ProviderUtils + .replaceSpacesWithDelimiter(launchCommand, ","); + } launcher.addCommand(launchCommand); } } else { diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/test/java/org/apache/hadoop/yarn/service/provider/TestProviderUtils.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/test/java/org/apache/hadoop/yarn/service/provider/TestProviderUtils.java index 0f7f375..3d93cbd 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/test/java/org/apache/hadoop/yarn/service/provider/TestProviderUtils.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/test/java/org/apache/hadoop/yarn/service/provider/TestProviderUtils.java @@ -168,4 +168,12 @@ public void setShouldBeUploadedToSharedCache( Assert.assertEquals(resolved.getResolvedRsrcPaths().get("destFile1"), "destFile1"); } + + @Test + public void testReplaceSpacesWithDelimiter() { + String command = "ls -l \" space\""; + String expected = "ls,-l, space"; + String actual = ProviderUtils.replaceSpacesWithDelimiter(command, ","); + Assert.assertEquals(expected, actual); + } }