commit 60e65105c3454598e3ea14c378652b1eb6c1f512 Author: Eric Yang Date: Mon Feb 25 17:44:42 2019 -0500 YARN-8805. Add ability to use space as delimiter in launch command. Contributed by Eric Yang diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/provider/ProviderUtils.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/provider/ProviderUtils.java index 88883f7..ea1fb0c 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/provider/ProviderUtils.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/provider/ProviderUtils.java @@ -47,9 +47,12 @@ import java.io.FileNotFoundException; import java.io.IOException; import java.io.OutputStream; +import java.util.ArrayList; import java.util.HashMap; +import java.util.List; import java.util.Map; import java.util.concurrent.ExecutionException; +import java.util.regex.Matcher; import java.util.regex.Pattern; import static org.apache.hadoop.yarn.service.api.ServiceApiConstants.COMPONENT_ID; @@ -149,6 +152,20 @@ public static String substituteStrWithTokens(String content, return content; } + public static String replaceSpacesWithDelimiter(String content, + String delimiter) { + List parts = new ArrayList(); + Matcher m = Pattern.compile("([^\"]\\S*|\".+?\")\\s*").matcher(content); + while (m.find()) { + String part = m.group(1); + if(part.startsWith("\"") && part.endsWith("\"")) { + part = part.replaceAll("^\"|\"$", ""); + } + parts.add(part); + } + return String.join(delimiter, parts); + } + // configs will be substituted by corresponding env in tokenMap public static void substituteMapWithTokens(Map configs, Map tokenMap) { diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/provider/docker/DockerProviderService.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/provider/docker/DockerProviderService.java index 6027a66..9b4138e 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/provider/docker/DockerProviderService.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/provider/docker/DockerProviderService.java @@ -25,6 +25,7 @@ import org.apache.hadoop.yarn.api.records.Container; import org.apache.hadoop.yarn.service.api.records.Service; import org.apache.hadoop.yarn.service.utils.SliderFileSystem; + import org.apache.hadoop.yarn.service.containerlaunch.AbstractLauncher; import org.apache.hadoop.yarn.service.containerlaunch.CommandLineBuilder; import org.apache.hadoop.yarn.service.containerlaunch.ContainerLaunchService; @@ -84,6 +85,11 @@ public void buildContainerLaunchCommand(AbstractLauncher launcher, if (useEntryPoint) { String launchCommand = compLaunchContext.getLaunchCommand(); if (!StringUtils.isEmpty(launchCommand)) { + if(launchCommand.contains(" ")) { + // convert space delimiter command to exec format + launchCommand = ProviderUtils + .replaceSpacesWithDelimiter(launchCommand, ","); + } launcher.addCommand(launchCommand); } } else { diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/test/java/org/apache/hadoop/yarn/service/provider/TestAbstractProviderService.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/test/java/org/apache/hadoop/yarn/service/provider/TestAbstractProviderService.java index 81ccc7f..ddb451a 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/test/java/org/apache/hadoop/yarn/service/provider/TestAbstractProviderService.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/test/java/org/apache/hadoop/yarn/service/provider/TestAbstractProviderService.java @@ -82,7 +82,7 @@ public void testBuildContainerLaunchCommand() throws Exception { Component component = serviceContext.scheduler.getAllComponents().entrySet() .iterator().next().getValue(); ContainerLaunchService.ComponentLaunchContext clc = - createEntryPointCLCFor(testService, component); + createEntryPointCLCFor(testService, component, "sleep,9000"); ComponentInstance instance = component.getAllComponentInstances().iterator() .next(); @@ -96,12 +96,31 @@ public void testBuildContainerLaunchCommand() throws Exception { } @Test + public void testBuildContainerLaunchCommandWithSpace() throws Exception { + AbstractProviderService providerService = new DockerProviderService(); + Component component = serviceContext.scheduler.getAllComponents().entrySet() + .iterator().next().getValue(); + ContainerLaunchService.ComponentLaunchContext clc = + createEntryPointCLCFor(testService, component, "ls -l \" space\""); + + ComponentInstance instance = component.getAllComponentInstances().iterator() + .next(); + Container container = mock(Container.class); + providerService.buildContainerLaunchCommand(launcher, testService, instance, + rule.getFs(), serviceContext.scheduler.getConfig(), container, clc, + null); + + Assert.assertEquals("commands don't match.", Lists.newArrayList("ls,-l, space"), + launcher.getCommands()); + } + + @Test public void testBuildContainerLaunchContext() throws Exception { AbstractProviderService providerService = new DockerProviderService(); Component component = serviceContext.scheduler.getAllComponents().entrySet() .iterator().next().getValue(); ContainerLaunchService.ComponentLaunchContext clc = - createEntryPointCLCFor(testService, component); + createEntryPointCLCFor(testService, component, "sleep,9000"); ComponentInstance instance = component.getAllComponentInstances().iterator() .next(); @@ -118,8 +137,8 @@ public void testBuildContainerLaunchContext() throws Exception { } private static ContainerLaunchService.ComponentLaunchContext - createEntryPointCLCFor(Service service, Component component) { - String launchCmd = "sleep,9000"; + createEntryPointCLCFor(Service service, Component component, + String launchCmd) { Artifact artifact = new Artifact(); artifact.setType(Artifact.TypeEnum.DOCKER); artifact.setId("example"); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/test/java/org/apache/hadoop/yarn/service/provider/TestProviderUtils.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/test/java/org/apache/hadoop/yarn/service/provider/TestProviderUtils.java index 0f7f375..f1cde82 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/test/java/org/apache/hadoop/yarn/service/provider/TestProviderUtils.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/test/java/org/apache/hadoop/yarn/service/provider/TestProviderUtils.java @@ -168,4 +168,12 @@ public void setShouldBeUploadedToSharedCache( Assert.assertEquals(resolved.getResolvedRsrcPaths().get("destFile1"), "destFile1"); } + + @Test + public void testReplaceSpacesWithDelimiter() { + String command = "ls -l \" space\""; + String expected = "ls,-l, space"; + String actual = ProviderUtils.replaceSpacesWithDelimiter(command, ","); + Assert.assertEquals("replaceSpaceWithDelimiter produces unexpected result.", expected, actual); + } }