diff --git a/hadoop-assemblies/src/main/resources/assemblies/hadoop-yarn-dist.xml b/hadoop-assemblies/src/main/resources/assemblies/hadoop-yarn-dist.xml index 8b3d292..83633ac 100644 --- a/hadoop-assemblies/src/main/resources/assemblies/hadoop-yarn-dist.xml +++ b/hadoop-assemblies/src/main/resources/assemblies/hadoop-yarn-dist.xml @@ -98,6 +98,13 @@ etc/hadoop + hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/examples + /share/hadoop/${hadoop.component}/yarn-service-examples + + **/* + + + hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/target /share/hadoop/${hadoop.component}/sources diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/examples/sleeper/description.txt b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/examples/sleeper/description.txt new file mode 100644 index 0000000..ad1714c --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/examples/sleeper/description.txt @@ -0,0 +1 @@ +Run a simple service that launches a few sleep containers. \ No newline at end of file diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/examples/sleeper/sleeper.json b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/examples/sleeper/sleeper.json new file mode 100644 index 0000000..89ce527 --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/examples/sleeper/sleeper.json @@ -0,0 +1,15 @@ +{ + "name": "sleeper-service", + "components" : + [ + { + "name": "sleeper", + "number_of_containers": 2, + "launch_command": "sleep 900000", + "resource": { + "cpus": 1, + "memory": "256" + } + } + ] +} \ No newline at end of file diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/ServiceCLI.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/ServiceCLI.java index 928c06f..48ff324 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/ServiceCLI.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/ServiceCLI.java @@ -20,14 +20,24 @@ import com.beust.jcommander.ParameterException; import org.apache.commons.lang.StringUtils; +import org.apache.hadoop.fs.FileUtil; +import org.apache.hadoop.yarn.api.ApplicationConstants; import org.apache.hadoop.yarn.conf.YarnConfiguration; +import org.apache.hadoop.yarn.exceptions.YarnException; import org.apache.hadoop.yarn.service.api.records.Service; import org.apache.hadoop.yarn.service.client.params.ClientArgs; import org.apache.hadoop.yarn.service.exceptions.SliderException; +import org.apache.hadoop.yarn.service.utils.ServiceApiUtil; import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import java.io.BufferedReader; +import java.io.File; +import java.io.FileInputStream; import java.io.FileNotFoundException; +import java.io.IOException; +import java.io.InputStreamReader; +import java.util.List; import static org.apache.hadoop.yarn.service.client.params.SliderActions.*; @@ -77,6 +87,9 @@ int exec(ClientArgs args) throws Throwable { client.updateLifetime(args.getClusterName(), args.getActionUpdateArgs().lifetime); break; + case ACTION_DEMO: + actionDemo(args); + break; case ACTION_HELP: LOG.info(args.usage()); break; @@ -88,6 +101,77 @@ int exec(ClientArgs args) throws Throwable { return 0; } + private void actionDemo(ClientArgs args) throws IOException, + YarnException { + List list = args.getActionDemoArgs().parameters; + // list[0] is service-type + // list[1] is service-name + if (list.size() != 2 || StringUtils.isEmpty(list.get(0)) || StringUtils + .isEmpty(list.get(1))) { + list_examples(); + } else { + File demoHome = getExampleHomeFolder(); + File jsonFile = + new File(demoHome, list.get(0) + "/" + list.get(0) + ".json"); + Service service; + try { + service = ServiceApiUtil.jsonSerDeser.fromFile(jsonFile); + } catch (FileNotFoundException e) { + System.err.println("Service type (" + list.get(0) + ") does not exist."); + return; + } + service.setName(list.get(1)); + client.actionCreate(service); + } + } + + private void list_examples() throws IOException { + StringBuilder builder = + new StringBuilder().append(System.lineSeparator()). + append("Run an example service") + .append(System.lineSeparator()) + .append("Usage: service demo ") + .append(System.lineSeparator()).append("Options:").append(System.lineSeparator()) + .append("\t\t\t The type of the service, which can be one of below: ") + .append(System.lineSeparator()); + + File examplesFolder = getExampleHomeFolder(); + File[] examples = FileUtil.listFiles(examplesFolder); + for (File f : examples) { + appendExample(f, builder); + } + builder.append(System.lineSeparator()).append(System.lineSeparator()) + .append("\t\t\t The name of the service"); + System.out.println(builder.toString()); + } + + private File getExampleHomeFolder() { + String yarnHome = + System.getenv(ApplicationConstants.Environment.HADOOP_YARN_HOME.key()); + return new File(yarnHome + "/share/hadoop/yarn/yarn-service-examples"); + } + + private void appendExample(File exampleFolder, StringBuilder builder) + throws IOException { + // append the service name + builder.append("\t\t").append(exampleFolder.getName()).append("\t\t"); + + // append the service description + try (FileInputStream fis = new FileInputStream( + new File(exampleFolder, "description.txt"))) { + BufferedReader bufferedReader = + new BufferedReader(new InputStreamReader(fis, "UTF-8")); + String firstLine = bufferedReader.readLine(); + if (!StringUtils.isEmpty(firstLine)) { + builder.append(firstLine); + } else { + builder.append("N/A"); + } + } catch (IOException e) { + System.err.println(e.getMessage()); + } + } + public ServiceCLI() { createServiceClient(); } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/params/ActionExamples.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/params/ActionDemoArgs.java similarity index 64% rename from hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/params/ActionExamples.java rename to hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/params/ActionDemoArgs.java index e489e17..244a1686 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/params/ActionExamples.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/params/ActionDemoArgs.java @@ -19,8 +19,19 @@ package org.apache.hadoop.yarn.service.client.params; import com.beust.jcommander.Parameters; +import org.apache.hadoop.yarn.service.exceptions.BadCommandArgumentsException; +import org.apache.hadoop.yarn.service.exceptions.UsageException; -@Parameters(commandNames = { SliderActions.ACTION_EXAMPLES}, - commandDescription = SliderActions.DESCRIBE_ACTION_EXAMPLES) -public class ActionExamples { +@Parameters(commandNames = { SliderActions.ACTION_DEMO }, + commandDescription = SliderActions.DESCRIBE_ACTION_DEMO) +public class ActionDemoArgs extends AbstractActionArgs{ + + @Override + public String getActionName() { + return SliderActions.ACTION_DEMO; + } + + @Override + public void validate() throws BadCommandArgumentsException, UsageException { + } } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/params/ClientArgs.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/params/ClientArgs.java index f479cd2..8e70db5 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/params/ClientArgs.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/params/ClientArgs.java @@ -19,8 +19,6 @@ package org.apache.hadoop.yarn.service.client.params; import org.apache.hadoop.conf.Configuration; -import org.apache.hadoop.yarn.conf.YarnConfiguration; -import org.apache.hadoop.yarn.service.conf.YarnServiceConf; import org.apache.hadoop.yarn.service.utils.SliderUtils; import org.apache.hadoop.yarn.service.exceptions.BadCommandArgumentsException; import org.apache.hadoop.yarn.service.exceptions.ErrorStrings; @@ -56,6 +54,7 @@ private final ActionThawArgs actionThawArgs = new ActionThawArgs(); private final ActionTokensArgs actionTokenArgs = new ActionTokensArgs(); private final ActionUpdateArgs actionUpdateArgs = new ActionUpdateArgs(); + private final ActionDemoArgs actionDemoArgs = new ActionDemoArgs(); public ClientArgs(String[] args) { super(args); @@ -72,7 +71,7 @@ protected void addActionArguments() { actionBuildArgs, actionCreateArgs, actionDependencyArgs, - actionDestroyArgs, + actionDestroyArgs, actionDemoArgs, actionFlexArgs, actionFreezeArgs, actionHelpArgs, @@ -149,6 +148,10 @@ public ActionTokensArgs getActionTokenArgs() { return actionTokenArgs; } + public ActionDemoArgs getActionDemoArgs() { + return actionDemoArgs; + } + /** * Look at the chosen action and bind it as the core action for the operation. * @throws SliderException bad argument or similar @@ -227,7 +230,9 @@ public void applyAction() throws SliderException { case ACTION_UPDATE: bindCoreAction(actionUpdateArgs); break; - + case ACTION_DEMO: + bindCoreAction(actionDemoArgs); + break; default: throw new BadCommandArgumentsException(ErrorStrings.ERROR_UNKNOWN_ACTION + " " + action); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/params/SliderActions.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/params/SliderActions.java index fa05f2e..beb4a2d 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/params/SliderActions.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/params/SliderActions.java @@ -30,7 +30,7 @@ String ACTION_UPGRADE = "upgrade"; String ACTION_DESTROY = "destroy"; String ACTION_EXISTS = "exists"; - String ACTION_EXAMPLES = "examples"; + String ACTION_DEMO = "demo"; String ACTION_FLEX = "flex"; String ACTION_STOP = "stop"; String ACTION_HELP = "help"; @@ -59,7 +59,7 @@ "Destroy a stopped service, service must be stopped first before destroying."; String DESCRIBE_ACTION_EXISTS = "Probe for a service running"; - String DESCRIBE_ACTION_EXAMPLES = "Run an example service on YARN"; + String DESCRIBE_ACTION_DEMO = "Run an example service on YARN"; String DESCRIBE_ACTION_FLEX = "Flex a service's component by increasing or decreasing the number of containers."; String DESCRIBE_ACTION_FREEZE = "Stop a running service"; diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/utils/JsonSerDeser.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/utils/JsonSerDeser.java index 7b22e3e..96faf34 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/utils/JsonSerDeser.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/utils/JsonSerDeser.java @@ -95,15 +95,9 @@ public T fromJson(String json) * @throws IOException IO problems * @throws JsonMappingException failure to map from the JSON to this class */ - public T fromFile(File jsonFile) - throws IOException, JsonParseException, JsonMappingException { + public T fromFile(File jsonFile) throws IOException { File absoluteFile = jsonFile.getAbsoluteFile(); - try { - return mapper.readValue(absoluteFile, classType); - } catch (IOException e) { - log.error("Exception while parsing json file {}", absoluteFile, e); - throw e; - } + return mapper.readValue(absoluteFile, classType); } /** diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/YarnCommands.md b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/YarnCommands.md index 8968f13..1e5e8bf 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/YarnCommands.md +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/YarnCommands.md @@ -77,28 +77,24 @@ Usage `yarn service [sub-command] [service-name] [options]` * `build`: Build a service with its specifications, but do not start it. ``` Usage: yarn service build --file [file] + Options: + --file,-f The local path to the service definition file ``` - | COMMAND\_OPTIONS | Description | - |:---- |:---- | - | --file or -f | The local path to the service definition file | - * `create`: create a service, it's equivalent to first invoke build and then start. ``` Usage: yarn service create --file [file] + Options: + --file or -f The local path to the service definition file ``` - | COMMAND\_OPTIONS | Description | - |:---- |:---- | - | --file or -f | The local path to the service definition file | - + * `dependency`: Yarn service framework dependency (libraries) management. ``` Usage: yarn service dependency [options] + Option: + --upload Pre-upload the dependency jars onto HDFS to expediate service launch process. ``` - | COMMAND\_OPTIONS | Description | - |:---- |:---- | - | --upload | Pre-upload the dependency jars onto HDFS to expediate service launch process. | - + * `destroy`: Destroy a stopped service, service must be stopped first before destroying. ``` Usage: yarn service destroy [service-name] @@ -106,10 +102,10 @@ Usage `yarn service [sub-command] [service-name] [options]` * `flex`: Flex a service's component by increasing or decreasing the number of containers. ``` Usage: yarn service flex [service-name] --component [component-name] [count] + Options: + --component [component-name] [count] + Specifies the component name and its number of containers. e.g. +1 incr by 1, -2 decr by 2, and 3 makes final count 3. ``` - | COMMAND\_OPTIONS | Description | - |:---- |:---- | - | --component [component-name] [count] | Specifies the component name and its number of containers. e.g. +1 incr by 1, -2 decr by 2, and 3 makes final count 3.| * `status`: Get the status of a service. ``` Usage: yarn service status [service-name] @@ -118,7 +114,13 @@ Usage `yarn service [sub-command] [service-name] [options]` ``` Usage: yarn service start [service-name] ``` - +* `demo`: Run an example service on YARN. + ``` + Usage: yarn service demo [service-type] [service-name] + Options: + [service-type] The type of the service, e.g. sleeper. + [service-name] The name of the service. + ``` ### `classpath` Usage: `yarn classpath [--glob |--jar |-h |--help]` diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/yarn-service/QuickStart.md b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/yarn-service/QuickStart.md index 327566b..901ab2a 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/yarn-service/QuickStart.md +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/yarn-service/QuickStart.md @@ -43,7 +43,14 @@ Below is a simple service definition that launches sleep containers on YARN by w ] } ``` - + User can simply run a pre-built example service on YARN using below command: +``` +yarn service demo [service-type] [service-name] +``` +e.g. Below command launches a `sleeper` service named as `my-sleeper` on YARN. +``` +yarn service demo sleeper my-sleeper +``` For launching docker based services using YARN Service framework, please refer to [API doc](YarnServiceAPI.md). ## Manage services on YARN via CLI