diff --git a/hadoop-project/pom.xml b/hadoop-project/pom.xml index c3366a5..0af7b1a 100755 --- a/hadoop-project/pom.xml +++ b/hadoop-project/pom.xml @@ -139,10 +139,10 @@ ${project.version} - + 1.5.4 1.16 - 1.30 + 1.48 diff --git a/hadoop-project/src/site/site.xml b/hadoop-project/src/site/site.xml index a4c40c4..b3259ff 100644 --- a/hadoop-project/src/site/site.xml +++ b/hadoop-project/src/site/site.xml @@ -149,10 +149,12 @@ - - - - + + + + + + diff --git a/hadoop-yarn-project/hadoop-yarn/bin/yarn b/hadoop-yarn-project/hadoop-yarn/bin/yarn index 43bcb6b..022287a 100755 --- a/hadoop-yarn-project/hadoop-yarn/bin/yarn +++ b/hadoop-yarn-project/hadoop-yarn/bin/yarn @@ -31,7 +31,7 @@ function hadoop_usage hadoop_add_option "--hosts filename" "list of hosts to use in worker mode" hadoop_add_option "--workers" "turn on worker mode" - hadoop_add_subcommand "apiserver" "run yarn-native-service rest server" + hadoop_add_subcommand "apiserver" daemon "run the api-server for deploying/managing services on YARN" hadoop_add_subcommand "application" client "prints application(s) report/kill application" hadoop_add_subcommand "applicationattempt" client "prints applicationattempt(s) report" hadoop_add_subcommand "classpath" client "prints the class path needed to get the hadoop jar and the required libraries" @@ -49,7 +49,7 @@ function hadoop_usage hadoop_add_subcommand "rmadmin" admin "admin tools" hadoop_add_subcommand "router" daemon "run the Router daemon" hadoop_add_subcommand "scmadmin" admin "SharedCacheManager admin tools" - hadoop_add_subcommand "service" "run a service" + hadoop_add_subcommand "service" client "deploy/manage a service on YARN" hadoop_add_subcommand "sharedcachemanager" admin "run the SharedCacheManager daemon" hadoop_add_subcommand "timelinereader" client "run the timeline reader server" hadoop_add_subcommand "timelineserver" daemon "run the timeline server" diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/main/java/org/apache/hadoop/yarn/service/webapp/ApiServer.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/main/java/org/apache/hadoop/yarn/service/webapp/ApiServer.java index e8286ef..edabf51 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/main/java/org/apache/hadoop/yarn/service/webapp/ApiServer.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/main/java/org/apache/hadoop/yarn/service/webapp/ApiServer.java @@ -149,7 +149,7 @@ private Response stopService(String appName, boolean destroy) { try { SERVICE_CLIENT.actionStop(appName, destroy); if (destroy) { - SERVICE_CLIENT.actionDestroy(appName); + SERVICE_CLIENT.actionDestroy(appName, false); LOG.info("Successfully deleted service {}", appName); } else { LOG.info("Successfully stopped service {}", appName); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/main/resources/definition/YARN-Simplified-V1-API-Layer-For-Services.yaml b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/main/resources/definition/YARN-Simplified-V1-API-Layer-For-Services.yaml index b084be7..b9b5b3a 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/main/resources/definition/YARN-Simplified-V1-API-Layer-For-Services.yaml +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/main/resources/definition/YARN-Simplified-V1-API-Layer-For-Services.yaml @@ -48,17 +48,16 @@ info: url: http://www.apache.org/licenses/LICENSE-2.0.html # the domain of the service host: host.mycompany.com +port: 9191(default) # array of all schemes that your API supports schemes: - http -# will be prefixed to all paths -basePath: /ws/v1/ consumes: - application/json produces: - application/json paths: - /services/version: + /ws/v1/services/version: get: summary: Get current version of the API server. description: Get current version of the API server. @@ -66,7 +65,7 @@ paths: 200: description: Successful request - /services: + /ws/v1/services: get: summary: (TBD) List of services running in the cluster. description: Get a list of all currently running services (response includes a minimal projection of the service info). For more details do a GET on a specific service name. @@ -103,7 +102,7 @@ paths: schema: $ref: '#/definitions/ServiceStatus' - /services/{service_name}: + /ws/v1/services/{service_name}: put: summary: Update a service or upgrade the binary version of the components of a running service description: Update the runtime properties of a service. Currently the following operations are supported - update lifetime, stop/start a service. @@ -174,7 +173,7 @@ paths: description: Unexpected error schema: $ref: '#/definitions/ServiceStatus' - /services/{service_name}/components/{component_name}: + /ws/v1/services/{service_name}/components/{component_name}: put: summary: Flex a component's number of instances. description: Set a component's desired number of instanes diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/ServiceMaster.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/ServiceMaster.java index e700961..2abdae1 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/ServiceMaster.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/ServiceMaster.java @@ -34,7 +34,7 @@ import org.apache.hadoop.yarn.conf.YarnConfiguration; import org.apache.hadoop.yarn.exceptions.YarnException; import org.apache.hadoop.yarn.security.client.ClientToAMTokenSecretManager; -import org.apache.hadoop.yarn.service.client.params.SliderAMArgs; +import org.apache.hadoop.yarn.service.client.params.ServiceAMArgs; import org.apache.hadoop.yarn.service.monitor.ServiceMonitor; import org.apache.hadoop.yarn.service.utils.ServiceApiUtil; import org.apache.hadoop.yarn.service.utils.SliderFileSystem; @@ -51,7 +51,7 @@ private static final Logger LOG = LoggerFactory.getLogger(ServiceMaster.class); - private static SliderAMArgs amArgs; + private static ServiceAMArgs amArgs; protected ServiceContext context; public ServiceMaster(String name) { @@ -108,7 +108,7 @@ protected ContainerId getAMContainerId() throws BadClusterStateException { } protected Path getAppDir() { - return new Path(amArgs.getAppDefPath()).getParent(); + return new Path(amArgs.getServiceDefPath()).getParent(); } protected ServiceScheduler createServiceScheduler(ServiceContext context) @@ -119,7 +119,7 @@ protected ServiceScheduler createServiceScheduler(ServiceContext context) protected void loadApplicationJson(ServiceContext context, SliderFileSystem fs) throws IOException { context.service = ServiceApiUtil - .loadServiceFrom(fs, new Path(amArgs.getAppDefPath())); + .loadServiceFrom(fs, new Path(amArgs.getServiceDefPath())); LOG.info(context.service.toString()); } @@ -138,7 +138,7 @@ private void printSystemEnv() { public static void main(String[] args) throws Exception { Thread.setDefaultUncaughtExceptionHandler(new YarnUncaughtExceptionHandler()); StringUtils.startupShutdownMessage(ServiceMaster.class, args, LOG); - amArgs = new SliderAMArgs(args); + amArgs = new ServiceAMArgs(args); amArgs.parse(); try { ServiceMaster serviceMaster = new ServiceMaster("Service Master"); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/ServiceCLI.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/ServiceCLI.java index cb27021..8e71ff9 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/ServiceCLI.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/ServiceCLI.java @@ -18,11 +18,12 @@ package org.apache.hadoop.yarn.service.client; +import com.beust.jcommander.ParameterException; import org.apache.commons.lang.StringUtils; import org.apache.hadoop.yarn.conf.YarnConfiguration; import org.apache.hadoop.yarn.service.api.records.Service; import org.apache.hadoop.yarn.service.client.params.ClientArgs; -import org.apache.hadoop.yarn.service.exceptions.BadCommandArgumentsException; +import org.apache.hadoop.yarn.service.exceptions.SliderException; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -67,7 +68,8 @@ int exec(ClientArgs args) throws Throwable { client.actionStop(args.getClusterName(), false); break; case ACTION_DESTROY: // Destroy can happen only if app is already stopped - client.actionDestroy(args.getClusterName()); + client.actionDestroy(args.getClusterName(), + args.getActionDestroyArgs().force); break; case ACTION_DEPENDENCY: // upload dependency jars client.actionDependency(args.getActionDependencyArgs()); @@ -101,7 +103,7 @@ public static void main(String[] args) throws Throwable { ClientArgs clientArgs = new ClientArgs(args); try { clientArgs.parse(); - } catch (BadCommandArgumentsException e) { + } catch (ParameterException | SliderException e) { System.err.println(e.getMessage()); System.exit(-1); } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/ServiceClient.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/ServiceClient.java index c3a2752..73591cf 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/ServiceClient.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/ServiceClient.java @@ -158,7 +158,7 @@ protected void serviceStop() throws Exception { private Service loadAppJsonFromLocalFS( AbstractClusterBuildingActionArgs args) throws IOException { - File file = args.getAppDef(); + File file = args.getFile(); Path filePath = new Path(file.getAbsolutePath()); LOG.info("Loading app json from: " + filePath); Service service = jsonSerDeser @@ -166,7 +166,6 @@ private Service loadAppJsonFromLocalFS( if (args.lifetime > 0) { service.setLifetime(args.lifetime); } - service.setName(args.getClusterName()); return service; } @@ -405,9 +404,11 @@ public int actionStop(String serviceName, boolean waitForAppStopped) return EXIT_SUCCESS; } - public int actionDestroy(String serviceName) throws Exception { + public int actionDestroy(String serviceName, boolean force) throws Exception { ServiceApiUtil.validateNameFormat(serviceName, getConfig()); - verifyNoLiveAppInRM(serviceName, "Destroy"); + if (!force) { + verifyNoLiveAppInRM(serviceName, "destroy"); + } Path appDir = fs.buildClusterDirPath(serviceName); FileSystem fileSystem = fs.getFileSystem(); // remove from the appId cache @@ -498,9 +499,16 @@ private void verifyNoLiveAppInRM(String serviceName, String action) request.setApplicationStates(liveStates); List reports = yarnClient.getApplications(request); if (!reports.isEmpty()) { - throw new YarnException( - "Failed to " + action + " service, as " + serviceName - + " already exists."); + String message = ""; + if (action.equals("destroy")) { + message = "Failed to destroy service " + serviceName + + ", because it is still active. Use --force option if you want" + + " to forcefully destroy it."; + } else { + message = "Failed to " + action + " service " + serviceName + + ", because it already exists."; + } + throw new YarnException(message); } } @@ -592,7 +600,7 @@ private String buildCommandLine(String serviceName, Configuration conf, CLI.add(ServiceMaster.class.getCanonicalName()); CLI.add(ACTION_CREATE, serviceName); //TODO debugAM CLI.add(Arguments.ARG_DEBUG) - CLI.add(Arguments.ARG_CLUSTER_URI, new Path(appRootDir, serviceName + ".json")); + CLI.add(Arguments.ARG_SERVICE_DEF_PATH, new Path(appRootDir, serviceName + ".json")); // pass the registry binding CLI.addConfOptionToCLI(conf, RegistryConstants.KEY_REGISTRY_ZK_ROOT, RegistryConstants.DEFAULT_ZK_REGISTRY_ROOT); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/params/AbstractActionArgs.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/params/AbstractActionArgs.java index ea3bb0a..8b2d93e 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/params/AbstractActionArgs.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/params/AbstractActionArgs.java @@ -41,22 +41,10 @@ protected AbstractActionArgs() { } /** - * URI/binding to the filesystem - */ - @Parameter(names = {ARG_FILESYSTEM, ARG_FILESYSTEM_LONG}, - description = "Filesystem Binding") - public String filesystemBinding; - - @Parameter(names = {ARG_BASE_PATH}, - description = "Service base path on the filesystem", - converter = PathArgumentConverter.class) - public Path basePath; - - /** * This is the default parameter */ @Parameter - public final List parameters = new ArrayList<>(); + public List parameters = new ArrayList<>(); /** * get the name: relies on arg 1 being the cluster name in all operations @@ -76,29 +64,22 @@ public String getClusterName() { */ - @Parameter(names = ARG_DEFINE, arity = 1, description = "Definitions") - public final List definitions = new ArrayList<>(); + @Parameter(names = ARG_DEFINE, arity = 1, description = "Definitions", hidden = true) + public List definitions = new ArrayList<>(); /** * System properties */ @Parameter(names = {ARG_SYSPROP}, arity = 1, description = "system properties in the form name value" + - " These are set after the JVM is started.") - public final List sysprops = new ArrayList<>(0); + " These are set after the JVM is started.", + hidden = true) + public List sysprops = new ArrayList<>(0); - @Parameter(names = {ARG_MANAGER_SHORT, ARG_MANAGER}, - description = "Binding (usually hostname:port) of the YARN resource manager") - public String manager; - - - @Parameter(names = ARG_DEBUG, description = "Debug mode") + @Parameter(names = ARG_DEBUG, description = "Debug mode", hidden = true) public boolean debug = false; - @Parameter(names = {ARG_HELP}, description = "Help", help = true) - public boolean help = false; - /** * Get the min #of params expected * @return the min number of params in the {@link #parameters} field diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/params/AbstractClusterBuildingActionArgs.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/params/AbstractClusterBuildingActionArgs.java index 3a3a19a..d215436 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/params/AbstractClusterBuildingActionArgs.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/params/AbstractClusterBuildingActionArgs.java @@ -19,13 +19,8 @@ package org.apache.hadoop.yarn.service.client.params; import com.beust.jcommander.Parameter; -import com.beust.jcommander.ParametersDelegate; -import com.google.common.annotations.VisibleForTesting; -import org.apache.hadoop.yarn.service.exceptions.BadCommandArgumentsException; import java.io.File; -import java.util.List; -import java.util.Map; /** * Abstract Action to build things; shares args across build and @@ -33,12 +28,12 @@ */ public abstract class AbstractClusterBuildingActionArgs extends AbstractActionArgs { - @Parameter(names = {ARG_APPDEF}, - description = "Template service definition file in JSON format.") - public File appDef; + @Parameter(names = { ARG_FILE, ARG_FILE_SHORT }, required = true, + description = "The path to the service definition file in JSON format.") + public File file; - public File getAppDef() { - return appDef; + public File getFile() { + return file; } @Parameter(names = { @@ -48,11 +43,4 @@ public File getAppDef() { @Parameter(names = { ARG_LIFETIME }, description = "Lifetime of the service from the time of request") public long lifetime; - - @ParametersDelegate - public ComponentArgsDelegate componentDelegate = new ComponentArgsDelegate(); - - @ParametersDelegate - public OptionArgsDelegate optionsDelegate = - new OptionArgsDelegate(); } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/params/ActionBuildArgs.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/params/ActionBuildArgs.java index c2ff545..cf61525 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/params/ActionBuildArgs.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/params/ActionBuildArgs.java @@ -28,4 +28,9 @@ public String getActionName() { return SliderActions.ACTION_BUILD; } + + @Override + public int getMinParams() { + return 0; + } } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/params/ActionCreateArgs.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/params/ActionCreateArgs.java index eecffb6..c43f4fc 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/params/ActionCreateArgs.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/params/ActionCreateArgs.java @@ -29,5 +29,10 @@ public String getActionName() { return SliderActions.ACTION_CREATE; } + + @Override + public int getMinParams() { + return 0; + } } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/params/ActionDependencyArgs.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/params/ActionDependencyArgs.java index 51e07c9..3bbd26d 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/params/ActionDependencyArgs.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/params/ActionDependencyArgs.java @@ -32,7 +32,7 @@ public String getActionName() { return SliderActions.ACTION_DEPENDENCY; } - @Parameter(names = { ARG_UPLOAD }, + @Parameter(names = { ARG_UPLOAD }, required = true, description = "Upload AM and agent libraries to HDFS for this client") public boolean upload; diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/params/ActionExamples.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/params/ActionExamples.java new file mode 100644 index 0000000..e489e17 --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/params/ActionExamples.java @@ -0,0 +1,26 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.yarn.service.client.params; + +import com.beust.jcommander.Parameters; + +@Parameters(commandNames = { SliderActions.ACTION_EXAMPLES}, + commandDescription = SliderActions.DESCRIBE_ACTION_EXAMPLES) +public class ActionExamples { +} diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/params/ActionUpdateArgs.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/params/ActionUpdateArgs.java index e310f45..00af69a 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/params/ActionUpdateArgs.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/params/ActionUpdateArgs.java @@ -18,12 +18,17 @@ package org.apache.hadoop.yarn.service.client.params; +import com.beust.jcommander.Parameter; import com.beust.jcommander.Parameters; @Parameters(commandNames = { SliderActions.ACTION_UPDATE}, commandDescription = SliderActions.DESCRIBE_ACTION_UPDATE) -public class ActionUpdateArgs extends AbstractClusterBuildingActionArgs { +public class ActionUpdateArgs extends AbstractActionArgs { + + @Parameter(names = { + ARG_LIFETIME }, description = "Lifetime of the service from the time of request") + public long lifetime; @Override public String getActionName() { diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/params/Arguments.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/params/Arguments.java index 204149b..911440d 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/params/Arguments.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/params/Arguments.java @@ -27,7 +27,8 @@ */ public interface Arguments { - String ARG_APPDEF = "--appdef"; + String ARG_FILE = "--file"; + String ARG_FILE_SHORT = "-f"; String ARG_BASE_PATH = "--basepath"; String ARG_COMPONENT = "--component"; String ARG_COMPONENT_SHORT = "--comp"; @@ -95,9 +96,7 @@ RIGHT PLACE IN THE LIST */ - /** - * server: URI for the cluster - */ - String ARG_CLUSTER_URI = "-cluster-uri"; + // Tha path in hdfs to be read by Service AM + String ARG_SERVICE_DEF_PATH = "-cluster-uri"; } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/params/ClientArgs.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/params/ClientArgs.java index 09cae24..f479cd2 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/params/ClientArgs.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/params/ClientArgs.java @@ -85,16 +85,6 @@ protected void addActionArguments() { public void applyDefinitions(Configuration conf) throws BadCommandArgumentsException { super.applyDefinitions(conf); - //RM - if (getManager() != null) { - log.debug("Setting RM to {}", getManager()); - conf.set(YarnConfiguration.RM_ADDRESS, getManager()); - } - if (getBasePath() != null) { - log.debug("Setting basePath to {}", getBasePath()); - conf.set(YarnServiceConf.YARN_SERVICE_BASE_PATH, - getBasePath().toString()); - } } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/params/CommonArgs.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/params/CommonArgs.java index e1197ea..ea78bcd 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/params/CommonArgs.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/params/CommonArgs.java @@ -156,12 +156,8 @@ public void parse() throws SliderException { try { commander.parse(args); } catch (ParameterException e) { - throw new BadCommandArgumentsException(e, "%s in %s", - e.toString(), - (args != null - ? (SliderUtils.join(args, - " ", false)) - : "[]")); + commander.usage(args[0]); + throw e; } //now copy back to this class some of the attributes that are common to all //actions @@ -235,6 +231,7 @@ public void validate() throws BadCommandArgumentsException, UsageException { try { coreAction.validate(); } catch (BadCommandArgumentsException e) { + String badArgMsgBuilder = e.getMessage() + System.lineSeparator() + usage(this, coreAction.getActionName()); @@ -251,31 +248,11 @@ public void applyDefinitions(Configuration conf) throws applyDefinitions(definitionMap, conf); } - - /** - * If the Filesystem binding was provided, it overrides anything in - * the configuration - * @param conf configuration - */ - public void applyFileSystemBinding(Configuration conf) { - ArgOps.applyFileSystemBinding(getFilesystemBinding(), conf); - } - public boolean isDebug() { return coreAction.debug; } - public String getFilesystemBinding() { - return coreAction.filesystemBinding; - } - - public Path getBasePath() { return coreAction.basePath; } - - public String getManager() { - return coreAction.manager; - } - public String getAction() { return commander.getParsedCommand(); } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/params/SliderAMArgs.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/params/ServiceAMArgs.java similarity index 73% rename from hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/params/SliderAMArgs.java rename to hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/params/ServiceAMArgs.java index 1c38213..800bb24 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/params/SliderAMArgs.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/params/ServiceAMArgs.java @@ -22,11 +22,11 @@ /** * Parameters sent by the Client to the AM */ -public class SliderAMArgs extends CommonArgs { +public class ServiceAMArgs extends CommonArgs { - SliderAMCreateAction createAction = new SliderAMCreateAction(); + ServiceAMCreateAction createAction = new ServiceAMCreateAction(); - public SliderAMArgs(String[] args) { + public ServiceAMArgs(String[] args) { super(args); } @@ -35,16 +35,9 @@ protected void addActionArguments() { addActions(createAction); } - public String getImage() { - return createAction.image; - } - - /** - * This is the URI in the FS to the Slider cluster; the conf file (and any - * other cluster-specifics) can be picked up here - */ - public String getAppDefPath() { - return createAction.sliderClusterURI; + // This is the path in hdfs to the service definition JSON file + public String getServiceDefPath() { + return createAction.serviceDefPath; } /** diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/params/SliderAMCreateAction.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/params/ServiceAMCreateAction.java similarity index 54% rename from hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/params/SliderAMCreateAction.java rename to hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/params/ServiceAMCreateAction.java index a446665..384e8ba 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/params/SliderAMCreateAction.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/params/ServiceAMCreateAction.java @@ -20,54 +20,18 @@ import com.beust.jcommander.Parameter; import com.beust.jcommander.Parameters; -import com.beust.jcommander.ParametersDelegate; - -import java.io.File; - @Parameters(commandNames = { SliderActions.ACTION_CREATE}, commandDescription = SliderActions.DESCRIBE_ACTION_CREATE) -public class SliderAMCreateAction extends AbstractActionArgs implements - LaunchArgsAccessor { - +public class ServiceAMCreateAction extends AbstractActionArgs { @Override public String getActionName() { return SliderActions.ACTION_CREATE; } - @Parameter(names = ARG_IMAGE, description = "image", required = false) - public String image; - - /** - * This is the URI in the FS to the Slider cluster; the conf file (and any - * other cluster-specifics) can be picked up here - */ - @Parameter(names = ARG_CLUSTER_URI, - description = "URI to the Slider cluster", required = true) - public String sliderClusterURI; - - @ParametersDelegate LaunchArgsDelegate launchArgs = new LaunchArgsDelegate(); - - @Override - public String getRmAddress() { - return launchArgs.getRmAddress(); - } - - @Override - public int getWaittime() { - return launchArgs.getWaittime(); - } - - @Override - public void setWaittime(int waittime) { - launchArgs.setWaittime(waittime); - } - - @Override - public File getOutputFile() { - return launchArgs.getOutputFile(); - } - + @Parameter(names = ARG_SERVICE_DEF_PATH, + description = "Path to the service definition JSON file", required = true) + public String serviceDefPath; } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/params/SliderActions.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/params/SliderActions.java index 1b2a92d..304c5ad 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/params/SliderActions.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/params/SliderActions.java @@ -19,9 +19,7 @@ package org.apache.hadoop.yarn.service.client.params; /** - * Actions. - * Only some of these are supported by specific Slider Services; they - * are listed here to ensure the names are consistent + * Actions by client */ public interface SliderActions { String ACTION_BUILD = "build"; @@ -32,11 +30,11 @@ String ACTION_UPGRADE = "upgrade"; String ACTION_DESTROY = "destroy"; String ACTION_EXISTS = "exists"; + String ACTION_EXAMPLES = "examples"; String ACTION_FLEX = "flex"; String ACTION_STOP = "stop"; String ACTION_HELP = "help"; String ACTION_INSTALL_KEYTAB = "install-keytab"; - String ACTION_KDIAG = "kdiag"; String ACTION_KEYTAB = "keytab"; String ACTION_LIST = "list"; @@ -50,7 +48,7 @@ String DESCRIBE_ACTION_BUILD = "Build a service specification, but do not start it"; String DESCRIBE_ACTION_CREATE = - "Build and start a service, it's equivalent to first invoke build and then start"; + "create a service, it's equivalent to first invoke build and then start"; String DESCRIBE_ACTION_DEPENDENCY = "Yarn service framework dependency (libraries) management"; String DESCRIBE_ACTION_UPDATE = @@ -61,6 +59,7 @@ "Destroy a stopped service, service must be stopped first before destroying."; String DESCRIBE_ACTION_EXISTS = "Probe for a service running"; + String DESCRIBE_ACTION_EXAMPLES = "Run example services on YARN"; String DESCRIBE_ACTION_FLEX = "Flex a service's component by increasing or decreasing the number of containers."; String DESCRIBE_ACTION_FREEZE = "Stop a running service"; diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/timelineservice/ServiceTimelinePublisher.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/timelineservice/ServiceTimelinePublisher.java index 9e7d8e8..cfe7c1b 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/timelineservice/ServiceTimelinePublisher.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/timelineservice/ServiceTimelinePublisher.java @@ -32,6 +32,7 @@ import org.apache.hadoop.yarn.service.api.records.ConfigFile; import org.apache.hadoop.yarn.service.api.records.Configuration; import org.apache.hadoop.yarn.service.api.records.Container; +import org.apache.hadoop.yarn.service.api.records.ServiceState; import org.apache.hadoop.yarn.service.component.instance.ComponentInstance; import org.apache.hadoop.yarn.util.timeline.TimelineUtils; import org.slf4j.Logger; @@ -96,8 +97,8 @@ public void serviceAttemptRegistered(Service service, // create info keys Map entityInfos = new HashMap(); entityInfos.put(ServiceTimelineMetricsConstants.NAME, service.getName()); -// entityInfos.put(ServiceTimelineMetricsConstants.STATE, -// service.getState().toString()); + entityInfos.put(ServiceTimelineMetricsConstants.STATE, + ServiceState.STARTED.toString()); entityInfos.put(ServiceTimelineMetricsConstants.LAUNCH_TIME, currentTimeMillis); entity.addInfo(ServiceTimelineMetricsConstants.QUICK_LINKS, diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/test/java/org/apache/hadoop/yarn/service/TestYarnNativeServices.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/test/java/org/apache/hadoop/yarn/service/TestYarnNativeServices.java index 30f2aeb..f16489e 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/test/java/org/apache/hadoop/yarn/service/TestYarnNativeServices.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/test/java/org/apache/hadoop/yarn/service/TestYarnNativeServices.java @@ -275,7 +275,7 @@ public void testCreateFlexStopDestroyService() throws Exception { LOG.info("Destroy the service"); //destroy the service and check the app dir is deleted from fs. - client.actionDestroy(exampleApp.getName()); + client.actionDestroy(exampleApp.getName(), false); // check the service dir on hdfs (in this case, local fs) are deleted. Assert.assertFalse(fs.exists(appDir)); } @@ -302,7 +302,7 @@ public void testComponentStartOrder() throws Exception { checkContainerLaunchDependencies(client, exampleApp, "compa", "compb"); client.actionStop(exampleApp.getName(), true); - client.actionDestroy(exampleApp.getName()); + client.actionDestroy(exampleApp.getName(), false); } // Check containers launched are in dependency order diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/test/java/org/apache/hadoop/yarn/service/client/TestBuildExternalComponents.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/test/java/org/apache/hadoop/yarn/service/client/TestBuildExternalComponents.java index 93a15cc..0498ebf 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/test/java/org/apache/hadoop/yarn/service/client/TestBuildExternalComponents.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/test/java/org/apache/hadoop/yarn/service/client/TestBuildExternalComponents.java @@ -36,7 +36,7 @@ import java.util.List; import java.util.Set; -import static org.apache.hadoop.yarn.service.client.params.Arguments.ARG_APPDEF; +import static org.apache.hadoop.yarn.service.client.params.Arguments.ARG_FILE; import static org.apache.hadoop.yarn.service.conf.YarnServiceConf.YARN_SERVICE_BASE_PATH; /** @@ -56,12 +56,12 @@ private static void checkComponentNames(List components, } } - // 1. Build the appDef and store on fs + // 1. Build the def file and store on fs // 2. check component names private void buildAndCheckComponents(String appName, String appDef, SliderFileSystem sfs, Set names) throws Throwable { String[] args = - { "build", appName, ARG_APPDEF, ExampleAppJson.resourceName(appDef) }; + { "build", ARG_FILE, ExampleAppJson.resourceName(appDef) }; ClientArgs clientArgs = new ClientArgs(args); clientArgs.parse(); ServiceCLI cli = new ServiceCLI() { diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/test/java/org/apache/hadoop/yarn/service/client/TestServiceCLI.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/test/java/org/apache/hadoop/yarn/service/client/TestServiceCLI.java index c53ee2b..f22d487 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/test/java/org/apache/hadoop/yarn/service/client/TestServiceCLI.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/test/java/org/apache/hadoop/yarn/service/client/TestServiceCLI.java @@ -24,7 +24,6 @@ import org.apache.hadoop.yarn.api.records.ApplicationReport; import org.apache.hadoop.yarn.api.records.YarnApplicationState; import org.apache.hadoop.yarn.conf.YarnConfiguration; -import org.apache.hadoop.yarn.exceptions.YarnException; import org.apache.hadoop.yarn.service.ClientAMProtocol; import org.apache.hadoop.yarn.service.api.records.Component; import org.apache.hadoop.yarn.service.client.params.ClientArgs; @@ -43,7 +42,7 @@ import static org.apache.hadoop.yarn.conf.YarnConfiguration.RESOURCEMANAGER_CONNECT_MAX_WAIT_MS; import static org.apache.hadoop.yarn.conf.YarnConfiguration.RESOURCEMANAGER_CONNECT_RETRY_INTERVAL_MS; -import static org.apache.hadoop.yarn.service.client.params.Arguments.ARG_APPDEF; +import static org.apache.hadoop.yarn.service.client.params.Arguments.ARG_FILE; import static org.apache.hadoop.yarn.service.conf.YarnServiceConf.YARN_SERVICE_BASE_PATH; import static org.mockito.Mockito.*; @@ -54,9 +53,9 @@ private ServiceCLI cli; private SliderFileSystem fs; - private void buildApp(String appName, String appDef) throws Throwable { + private void buildApp(String appDef) throws Throwable { String[] args = - { "build", appName, ARG_APPDEF, ExampleAppJson.resourceName(appDef) }; + { "build", ARG_FILE, ExampleAppJson.resourceName(appDef)}; ClientArgs clientArgs = new ClientArgs(args); clientArgs.parse(); cli.exec(clientArgs); @@ -115,34 +114,35 @@ public void tearDown() throws IOException { // Test flex components count are persisted. @Test public void testFlexComponents() throws Throwable { - buildApp("service-1", ExampleAppJson.APP_JSON); - - checkCompCount("master", 1L); + String serviceName = "app-1"; + buildApp(ExampleAppJson.APP_JSON); + checkCompCount("master",serviceName, 1L); // increase by 2 - String[] flexUpArgs = {"flex", "service-1", "--component", "master" , "+2"}; + String[] flexUpArgs = {"flex", serviceName, "--component", "master" , "+2"}; ClientArgs clientArgs = new ClientArgs(flexUpArgs); clientArgs.parse(); cli.exec(clientArgs); - checkCompCount("master", 3L); + checkCompCount("master", serviceName, 3L); // decrease by 1 - String[] flexDownArgs = {"flex", "service-1", "--component", "master", "-1"}; + String[] flexDownArgs = {"flex", serviceName, "--component", "master", "-1"}; clientArgs = new ClientArgs(flexDownArgs); clientArgs.parse(); cli.exec(clientArgs); - checkCompCount("master", 2L); + checkCompCount("master", serviceName, 2L); - String[] flexAbsoluteArgs = {"flex", "service-1", "--component", "master", "10"}; + String[] flexAbsoluteArgs = {"flex", serviceName, "--component", "master", "10"}; clientArgs = new ClientArgs(flexAbsoluteArgs); clientArgs.parse(); cli.exec(clientArgs); - checkCompCount("master", 10L); + checkCompCount("master", serviceName, 10L); } - private void checkCompCount(String compName, long count) throws IOException { + private void checkCompCount(String compName, String serviceName, long count) + throws IOException { List components = - ServiceApiUtil.getComponents(fs, "service-1"); + ServiceApiUtil.getComponents(fs, serviceName); for (Component component : components) { if (component.getName().equals(compName)) { Assert.assertEquals(count, component.getNumberOfContainers().longValue()); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/test/resources/org/apache/hadoop/yarn/service/conf/examples/app.json b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/test/resources/org/apache/hadoop/yarn/service/conf/examples/app.json index 12b51e4..43d3c39 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/test/resources/org/apache/hadoop/yarn/service/conf/examples/app.json +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/test/resources/org/apache/hadoop/yarn/service/conf/examples/app.json @@ -1,5 +1,5 @@ { - "name": "service-1", + "name": "app-1", "id" : "application_1503358878042_0011", "lifetime": "3600", "launch_command": "sleep 3600", diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-registry/src/main/java/org/apache/hadoop/registry/client/api/RegistryConstants.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-registry/src/main/java/org/apache/hadoop/registry/client/api/RegistryConstants.java index e66a761..0006dfd 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-registry/src/main/java/org/apache/hadoop/registry/client/api/RegistryConstants.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-registry/src/main/java/org/apache/hadoop/registry/client/api/RegistryConstants.java @@ -95,7 +95,7 @@ /** * Default DNS port number. */ - int DEFAULT_DNS_PORT = 53; + int DEFAULT_DNS_PORT = 5353; /** * DNSSEC Enabled? diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/YarnCommands.md b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/YarnCommands.md index 5f430ec..4fabb57 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/YarnCommands.md +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/YarnCommands.md @@ -22,7 +22,7 @@ Overview YARN commands are invoked by the bin/yarn script. Running the yarn script without any arguments prints the description for all commands. -Usage: `yarn [SHELL_OPTIONS] COMMAND [GENERIC_OPTIONS] [COMMAND_OPTIONS]` +Usage: `yarn [SHELL_OPTIONS] COMMAND [GENERIC_OPTIONS] [SUB_COMMAND] [COMMAND_OPTIONS]` YARN has an option parsing framework that employs parsing generic options as well as running classes. @@ -69,6 +69,56 @@ Usage: `yarn applicationattempt [options] ` prints applicationattempt(s) report +### `service` +Usage `yarn service [sub-command] [service-name] [options]` + +#### `SUB_COMMAND` + +* `build`: Build a service with its specifications, but do not start it. + ``` + Usage: yarn service build --file [file] + ``` + + | COMMAND\_OPTIONS | Description | + |:---- |:---- | + | --file or -f | The local path to the service definition file | + +* `create`: create a service, it's equivalent to first invoke build and then start. + ``` + Usage: yarn service create --file [file] + ``` + | COMMAND\_OPTIONS | Description | + |:---- |:---- | + | --file or -f | The local path to the service definition file | + +* `dependency`: Yarn service framework dependency (libraries) management. + ``` + Usage: yarn service dependency [options] + ``` + | COMMAND\_OPTIONS | Description | + |:---- |:---- | + | --upload | Pre-upload the dependency jars onto HDFS to expediate service launch process. | + +* `destroy`: Destroy a stopped service, service must be stopped first before destroying. + ``` + Usage: yarn service destroy [service-name] + ``` +* `flex`: Flex a service's component by increasing or decreasing the number of containers. + ``` + Usage: yarn service flex [service-name] --component [component-name] [count] + ``` + | COMMAND\_OPTIONS | Description | + |:---- |:---- | + | --component [component-name] [count] | Specifies the component name and its number of containers. e.g. +1 incr by 1, -2 decr by 2, and 3 makes final count 3.| +* `status`: Get the status of a service. + ``` + Usage: yarn service status [service-name] + ``` +* `start`: Start a service with pre-built specification or a previously stopped service. + ``` + Usage: yarn service start [service-name] + ``` + ### `classpath` Usage: `yarn classpath [--glob |--jar |-h |--help]` @@ -260,6 +310,10 @@ Usage: `yarn timelineserver` Start the TimeLineServer +### apiserver +Usage: `yarn apiserver` +Start the API-server for deploying/managing services on YARN + Files ----- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/native-services/NativeServicesIntro.md b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/native-services/NativeServicesIntro.md deleted file mode 100644 index e6a4e91..0000000 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/native-services/NativeServicesIntro.md +++ /dev/null @@ -1,107 +0,0 @@ - - -# Introduction: YARN Native Services - -## Overview -YARN Native Services provides first class framework support and APIs to host long running services natively in YARN. In addition to launching services, the new APIs support performing lifecycle management operations, such as flex service components up/down, manage lifetime, upgrade the service to a newer version, and stop/restart/delete the service. - -The native services capabilities are built on the existing low-level resource management API provided by YARN that can support any type of application. Other application frameworks like Hadoop MapReduce already expose higher level APIs that users can leverage to run applications on top of YARN. With the advent of containerization technologies like Docker, providing first class support and APIs for long running services at the framework level made sense. - -Relying on a framework has the advantage of exposing a simpler usage model to the user by enabling service configuration and launch through specification (without writing new code), as well as hiding complex low-level details including state management and fault-tolerance etc. Users/operators of existing services typically like to avoid modifying an existing service to be aware of YARN. With first class support capable of running a single Docker image as well as complex assemblies comprised of multiple Docker images, there is no need for service owners to be aware of YARN. Developers of new services do not have to worry about YARN internals and only need to focus on containerization of their service(s). - -## First class support for services -In order to natively provide first class support for long running services, several new features and improvements have been made at the framework level. - -### Incorporate Apache Slider into Apache YARN -Apache Slider, which existed as a separate incubator project has been merged into YARN to kick start the first class support. Apache Slider is a universal Application Master (AM) which had several key features built in - fault tolerance of service containers and AM, work-preserving AM restarts, service logs management, service management like flex up/down, stop/start, and rolling upgrade to newer service versions, etc. Of course lot more work has been done on top of what Apache Slider brought in, details of which follow. - -### Native Services API -A significant effort has gone into simplifying the user facing story for building services. In the past, bringing a new service to YARN was not a pleasant experience. The APIs of existing frameworks are either too low-level (native YARN), require writing new code (for frameworks with programmatic APIs) or require writing a complex spec (for declarative frameworks). - -The new REST APIs are very simple to use. The REST layer acts as a single point of entry for creation and lifecycle management of YARN services. Services here can range from simple single-component apps to the most complex, multi-component applications needing special orchestration needs. - -Plan is to make this a unified REST based entry point for other important features like resource-profile management ([YARN-3926](https://issues.apache.org/jira/browse/YARN-4793)), package-definitions' lifecycle-management and service-discovery ([YARN-913](https://issues.apache.org/jira/browse/YARN-913)/[YARN-4757](https://issues.apache.org/jira/browse/YARN-4757)). - -### Native Services Discovery -The new discovery solution exposes the registry information through a more generic and widely used mechanism: DNS. Service Discovery via DNS uses the well-known DNS interfaces to browse the network for services. Having the registry information exposed via DNS simplifies the life of services. - -The previous read mechanisms of YARN Service Registry were limited to a registry specific (java) API and a REST interface. In practice, this made it very difficult for wiring up existing clients and services. For e.g., dynamic configuration of dependent endpoints of a service was not easy to implement using the registry-read mechanisms, **without** code-changes to existing services. These are solved by the DNS based service discovery. - -### Scheduling -[YARN-6592](https://issues.apache.org/jira/browse/YARN-6592) covers a host of scheduling features that are useful for short-running applications and services alike. Below, are a few very important YARN core features that help schedule services better. Without these, running services on YARN is a hassle. - -* Affinity (TBD) -* Anti-affinity (TBD) -* Gang scheduling (TBD) -* Malleable container sizes ([YARN-1197](https://issues.apache.org/jira/browse/YARN-1197)) - -### Resource Profiles -YARN always had support for memory as a resource, inheriting it from Hadoop-(1.x)’s MapReduce platform. Later support for CPU as a resource ([YARN-2](https://issues.apache.org/jira/browse/YARN-2)/[YARN-3](https://issues.apache.org/jira/browse/YARN-3)) was added. Multiple efforts added support for various other resource-types in YARN such as disk ([YARN-2139](https://issues.apache.org/jira/browse/YARN-2139)), and network ([YARN-2140](https://issues.apache.org/jira/browse/YARN-2140)), specifically benefiting long running services. - -In many systems outside of YARN, users are already accustomed to specifying their desired ‘box’ of requirements where each box comes with a predefined amount of each resources. Admins would define various available box-sizes (small, medium, large etc) and users would pick the ones they desire and everybody is happy. In [YARN-3926](https://issues.apache.org/jira/browse/YARN-3926), YARN introduces Resource Profiles which extends the YARN resource model for easier resource-type management and profiles. This helps in two ways - the system can schedule applications better and it can perform intelligent over-subscription of resources where applicable. - -Resource profiles are all the more important for services since - -* Similar to short running apps, you don’t have to fiddle with varying resource-requirements for each container type -* Services usually end up planning for peak usages, leaving a lot of possibility of barren utilization - -### Special handling of preemption and container reservations -TBD. - -Preemption and reservation of long running containers have different implications from regular ones. Preemption of resources in YARN today works by killing of containers. For long-lived services this is unacceptable. Also, scheduler should avoid allocating long running containers on borrowed resources. [YARN-4724](https://issues.apache.org/jira/browse/YARN-4724) will address some of these special recognition of service containers. - -### Container auto-restarts -If a service container dies, expiring container's allocation and releasing the allocation is undesirable in many cases. Long running containers may exit for various reasons, crash and need to restart but forcing them to go through the complete scheduling cycle, resource localization, etc. is both unnecessary and expensive. - -Services can enable app-specific policies to prevent NodeManagers to automatically restart containers. [YARN-3998](https://issues.apache.org/jira/browse/YARN-3998) implements a retry-policy to let NM re-launch a service container when it fails. - -### Container allocation re-use for application upgrades -TBD. - -Auto-restart of containers will support upgrade of service containers without reclaiming the resources first. During an upgrade, with multitude of other applications running in the system, giving up and getting back resources allocated to the service is hard to manage. Node-Labels help this cause but are not straight-forward to use to address the app-specific use-cases. The umbrella [YARN-4726](https://issues.apache.org/jira/browse/YARN-4726) along with [YARN-5620](https://issues.apache.org/jira/browse/YARN-5620) and [YARN-4470](https://issues.apache.org/jira/browse/YARN-4470) will take care of this. - -### Dynamic Configurations -Most production-level services require dynamic configurations to manage and simplify their lifecycle. Container’s resource size, local/work dirs and log-dirs are the most basic information services need. Service's endpoint details (host/port), their inter-component dependencies, health-check endpoints, etc. are all critical to the success of today's real-life services. - -### Resource re-localization for reconfiguration/upgrades -TBD - -### Service Registry -TBD - -### Service persistent storage and volume support -TBD - -### Packaging -TBD - -### Container image registry (private, public and hybrid) -TBD - -### Container image management and APIs -TBD - -### Container image storage -TBD - -### Monitoring -TBD - -### Metrics -TBD - -### Service Logs -TBD - - diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/yarn-service/Concepts.md b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/yarn-service/Concepts.md new file mode 100644 index 0000000..716ff5f --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/yarn-service/Concepts.md @@ -0,0 +1,78 @@ + + +# Concepts +This document describes some key concepts and features that makes YARN as a first-class platform in order to natively support long running services on YARN. + +### Service Framework (ApplicationMaster) on YARN +A container orchestration framework is implemented to help deploying services on YARN. In a nutshell, the framework is an ApplicationMaster that +requests containers from ResourceManager based on service definition provided by the user and launch the containers across the cluster adhering to placement policies. +It also does all the heavy lifting work such as resolving the service definition and configurations, managing component life cycles such as automatically restarting +failed containers, monitoring components' healthiness and readiness, ensuring dependency start order across components, flexing up/down components, +upgrading components etc. The end goal of the framework is to make sure the service is up and running as the state that user desired. + + +### A Restful API-Server for deploying/managing services on YARN +A restful API server is developed to allow users to deploy/manage their services on YARN via a simple JSON spec. This avoids users +from dealing with the low-level APIs, writing complex code to bring their services onto YARN. The REST layer acts as a unified REST based entry for +creation and lifecycle management of YARN services. Services here can range from simple single-component apps to the most complex, +multi-component applications needing special orchestration needs. Please refer to this [API doc](YarnServiceAPI.md) for detailed API documentations. + +The API-server is stateless, which means users can simply spin up multiple instances, and have a load balancer fronting them to +support HA, distribute the load etc. + +### Service Discovery +A DNS server is implemented to enable discovering services on YARN via the standard mechanism: DNS lookup. +The DNS server essentially exposes the information in YARN service registry by translating them into DNS records such as A record and SRV record. +Clients can discover the IPs of containers via standard DNS lookup. +The previous read mechanisms of YARN Service Registry were limited to a registry specific (java) API and a REST interface and are difficult +to wireup existing clients and services. The DNS based service discovery eliminates this gap. Please refer to this [DNS doc](ServiceDiscovery.md) +for more details. + +### Scheduling + +A host of scheduling features are being developed to support long running services. + +* Affinity and anti-affinity scheduling across containers ([YARN-6592](https://issues.apache.org/jira/browse/YARN-6592)). +* Container resizing ([YARN-1197](https://issues.apache.org/jira/browse/YARN-1197)) +* Special handling of container preemption/reservation for services + +### Container auto-restarts + +[YARN-3998](https://issues.apache.org/jira/browse/YARN-3998) implements a retry-policy to let NM re-launch a service container when it fails. +The service REST API provides users a way to enable NodeManager to automatically restart the container if it fails. +The advantage is that it avoids the entire cycle of releasing the failed containers, re-asking new containers, re-do resource localizations and so on, which +greatly minimizes container downtime. + + +### Container in-place upgrade + +[YARN-4726](https://issues.apache.org/jira/browse/YARN-4726) supported upgrading containers in-place, that is, without losing the container allocations. +It opens up a few APIs in NodeManager to allow ApplicationMasters to upgrade their containers via a simple API call. +Under the hood, NodeManager does below steps: +* Downloading the new resources such as jars, docker container images, new configurations. +* Stop the old container. +* Start the new container with the newly downloaded resources. + +At the time of writing this document, YARN core side changes is complete. What's left is to hook up with the service REST API and the framework +to enable users to upgrade their service containers via a simple JSON spec. + +### Resource Profiles + +In [YARN-3926](https://issues.apache.org/jira/browse/YARN-3926), YARN introduces Resource Profiles which extends the YARN resource model for easier +resource-type management and profiles. +It primarily solves two problems: +* Make it easy to support new resource types such as network bandwith([YARN-2140](https://issues.apache.org/jira/browse/YARN-2140)), disks([YARN-2139](https://issues.apache.org/jira/browse/YARN-2139)). + Under the hood, it unifies the scheduler codebase to essentially parameterize the resource types. +* User can specify the container resource requirement by a profile name, rather than fiddling with varying resource-requirements for each resource type. diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/yarn-service/Overview.md b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/yarn-service/Overview.md new file mode 100644 index 0000000..5cc9c04 --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/yarn-service/Overview.md @@ -0,0 +1,51 @@ + + +# YARN Service +## Overview +Yarn Service framework provides first class support and APIs to host long running services natively in YARN. +In a nutshell, it serves as a container orchestration platform for managing containerized services on YARN. It supports both docker container +and traditional process based containers in YARN. + +The responsibility of this framework includes performing configuration resolutions and mounts, +lifecycle management such as stop/start/delete the service, flexing service components up/down, rolling upgrades services on YARN, monitoring services' healthiness and readiness and more. + +The yarn-service framework primarily includes below components: + +* A core framework (ApplicationMaster) running on YARN to serve as a container orchestrator, being responsible for all service lifecycle managements. +* A restful API-server to for users to interact with YARN to deploy/manage their services via a simple JSON spec. +* A DNS server backed by YARN service registry to enable discovering services on YARN by the standard DNS lookup. + +##Contents + +* [QuickStart](QuickStart.md): A tutorial to walk you through the steps to deploy a service on YARN. +* [Concepts](Concepts.md): Describes the internals of the framework and some features in YARN core to support running services on YARN. +* [Service Discovery](ServiceDiscovery.md): Deep dives into the YARN DNS internals. +* [Service REST API](YarnServiceAPI.md): The API doc for deploying/managing services on YARN. + +This feature is still in alpha state. We will continously update the documents as feature matures. + +## Reasons why you should consider this feature + +* This feature makes users' life extremely easier to bring existing services onto YARN. It hides all the complex low-level +details to eliminate users from writing new code. Developers of new services do not have to worry about YARN internals and + only need to focus on containerization of their service(s). + +Beyond, another huge win of this feature is that now you can enable both traditional batch processing jobs which YARN already well supports and long running services in a single platform ! +The benefits are mainly two-fold: + +* Greatly simplify the cluster operations as you have only a single cluster to deal with. +* Making both batch jobs and services share a cluster can greatly improve resource utilizations. + + diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/yarn-service/QuickStart.md b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/yarn-service/QuickStart.md new file mode 100644 index 0000000..37dac23 --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/yarn-service/QuickStart.md @@ -0,0 +1,202 @@ + + +# Quick Start + +This document describes how to deploy services on YARN using the Yarn-Service framework. + + + +## Start HDFS and YARN components + + Start all the hadoop components HDFS, YARN as usual. + + +## Example service +Below is a simple service definition that launches sleep containers on YARN without using docker. +You can use this for the following exercises. For docker based services, please refer to [API doc](YarnServiceAPI.md) for the time being. +``` +{ + "name": "sleep-service", + "components" : + [ + { + "name": "sleep", + "number_of_containers": 1, + "launch_command": "sleep 900000", + "resource": { + "cpus": 1, + "memory": "256" + } + } + ] +} +``` + +## Manage services on YARN via CLI +Below steps walk you through deploying a services on YARN using CLI. +Refer to [Yarn Commands](../YarnCommands.md) for the full list of commands and options. +### Deploy a service +``` +yarn service create --file ${PATH_TO_SERVICE_DEF_FILE} +``` +Params: +- SERVICE_NAME: The name of the service. Note that this needs to be unique across all running services. +- PATH_TO_SERVICE_DEF: The path to the service definition file in JSON format. + +For example: +``` +yarn service create --file /path/to/local/sleep.json +``` + +### Flex a component of a service +Increase or decrease the number of containers for a component. +``` +yarn service flex ${SERVICE_NAME} --component ${COMPONENT_NAME} ${NUMBER_OF_CONTAINERS} +``` +For example, for a service named `sleep-service`: + +Set the `sleep` component to `2` containers (absolute number). + +``` +yarn service flex sleep-service --component sleep 2 +``` + +Increase the `sleep` component by `2` containers. +``` +yarn service flex sleep-service --component sleep +2 +``` +Decrease the `sleep` component by `2` containers. +``` +yarn service flex sleep-service --component sleep -2 +``` + +### Stop a service +Stopping a service will stop all containers of the service and the ApplicationMaster, but does not delete the state of a service, such as the service root folder on hdfs. +``` +yarn service stop ${SERVICE_NAME} +``` + +### Destroy a service +In addition to stopping a service, it also deletes the service root folder on hdfs and the records in Yarn Service Registry. +``` +yarn service destroy ${SERVICE_NAME} +``` + +## Manage services on YARN via REST API +Below steps walk you through deploying services on YARN via REST API. + Refer to [API doc](YarnServiceAPI.md) for the detailed API specificatiosn. +### Start API-Server for deploying services on YARN +``` + yarn --daemon start apiserver + ``` + +### Deploy a service +POST the aforementioned example service definition to the api-server endpoint: +``` +POST http://localhost:9191/ws/v1/services +``` + +### Get a service status +``` +GET http://localhost:9191/ws/v1/services/${SERVICE_NAME} +``` + +### Flex a component of a service +``` +PUT http://localhost:9191/ws/v1/services/${SERVICE_NAME}/components/${COMPONENT_NAME} +``` +`PUT` Request body: +``` +{ + "name": "${COMPONENT_NAME}", + "number_of_containers": ${COUNT} +} +``` +For example: +``` +{ + "name": "sleep", + "number_of_containers": 2 +} +``` + +### Stop a service +Stopping a service will stop all containers of the service and the ApplicationMaster, but does not delete the state of a service, such as the service root folder on hdfs. + +``` +PUT http://locahost/ws/v1/services/${SERVICE_NAME} +``` + +`PUT` Request body: +``` +{ + "name": "${SERVICE_NAME}", + "state": "STOPPED" +} +``` +### Destroy a service +In addition to stopping a service, it also deletes the service root folder on hdfs and the records in Yarn Service Registry. +``` +DELETE http://localhost:9191/ws/v1/services/${SERVICE_NAME} +``` + +## New YARN Service UI with Timeline Service v2 +A new `service` tab is added in the new YARN UI. +The service framework (ApplicationMaster) posts the data into TimelineService and the `service` UI reads data from TimelineService to render its content. + +### Enable Timeline Service v2 +Please refer to [TimeLineService v2 doc](../TimelineServiceV2.md) for how to enable Timeline Service v2. + +### Enable new YARN UI + +Set below config in `yarn-site.xml` and start ResourceManager. +If you are building from source code, make sure you use `-Pyarn-ui` in the `mvn` command - this will generate the war file for the new YARN UI. +``` + + To enable RM web ui2 application. + yarn.webapp.ui2.enable + true + +``` + +## Service Discovery with YARN DNS +A DNS server (backed by YARN Service Registry) is implemented to enable the use case of discovering services deployed on YARN via DNS. +That is, user can simply access their services in a well-defined naming format as below: + +``` +${COMPONENT_INSTANCE_NAME}.${SERVICE_NAME}.${USER}.${DOMAIN} +``` +For example, in a cluster whose domain name is `yarncluster` (as defined by the `hadoop.registry.dns.domain-name` in `yarn-site.xml`), a service named `hbase` deployed by user `dev` +with two components `hbasemaster` and `regionserver` can be accessed as below: + +This URL points to the usual hbase master UI +``` +http://hbasemaster-0.hbase.dev.yarncluster:16010/master-status +``` + + +Note that YARN service framework assigns COMPONENT_INSTANCE_NAME for each container in a sequence of monotonically increasing integers. For example, `hbasemaster-0` gets +assigned `0` since it is the first and only instance for the `hbasemaster` component. In case of `regionserver` component, it can have multiple containers + and so be named as such: `regionserver-0`, `regionserver-1`, `regionserver-2` ... etc + +`Disclaimer`: The DNS implementation is still experimental. It should not be used as a full-functional corporate DNS. + +### Start the DNS server +By default, the DNS runs on non-privileged port `5353`. +If it is configured to use the standard privileged port `53`, the DNS server needs to be run as root: +``` +sudo su - -c "yarn org.apache.hadoop.registry.server.dns.RegistryDNSServer > /${HADOOP_LOG_FOLDER}/registryDNS.log 2>&1 &" root +``` +Please refer to [YARN DNS doc](ServicesDiscovery.md) for the full list of configurations. \ No newline at end of file diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/native-services/NativeServicesDiscovery.md b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/yarn-service/ServiceDiscovery.md similarity index 100% rename from hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/native-services/NativeServicesDiscovery.md rename to hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/yarn-service/ServiceDiscovery.md diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/native-services/NativeServicesAPI.md b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/yarn-service/YarnServiceAPI.md similarity index 93% rename from hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/native-services/NativeServicesAPI.md rename to hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/yarn-service/YarnServiceAPI.md index f56139a..4551e2b 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/native-services/NativeServicesAPI.md +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/yarn-service/YarnServiceAPI.md @@ -12,34 +12,11 @@ limitations under the License. See accompanying LICENSE file. --> -# YARN Simplified API layer for services - -## Overview -Bringing a new service on YARN today is not a simple experience. The APIs of -existing frameworks are either too low level (native YARN), require writing -new code (for frameworks with programmatic APIs) or writing a complex spec -(for declarative frameworks). In addition to building critical building blocks -inside YARN (as part of other efforts at -[YARN-4692](https://issues.apache.org/jira/browse/YARN-4692)), there is a need for -simplifying the user facing story for building services. Experience of projects -like Apache Slider running real-life services like HBase, Storm, Accumulo, -Solr etc, gives us some very good insights on how simplified APIs for services -should look like. - -To this end, we should look at a new simple-services API layer backed by REST -interfaces. This API can be used to create and manage the lifecycle of YARN -services. Services here can range from simple single-component service to -complex multi-component assemblies needing orchestration. -[YARN-4793](https://issues.apache.org/jira/browse/YARN-4793) tracks this -effort. - -This document spotlights on this specification. In most of the cases, the -application owner will not be forced to make any changes to their applications. -This is primarily true if the application is packaged with containerization -technologies like docker. Irrespective of how complex the application is, -there will be hooks provided at appropriate layers to allow pluggable and -customizable application behavior. +# YARN Service API +This document describes the API specifications (aka. YarnFile) for deploying/managing +containerized services on YARN. The same JSON spec can be used for both REST API +and CLI to manage the services. ### Version information Version: 1.0.0 @@ -51,7 +28,7 @@ License URL: http://www.apache.org/licenses/LICENSE-2.0.html ### URI scheme Host: host.mycompany.com -BasePath: /ws/v1/ +Port: 9191(default) Schemes: HTTP @@ -68,7 +45,7 @@ Schemes: HTTP ## Paths ### Create a service ``` -POST /services +POST /ws/v1/services ``` #### Description @@ -92,7 +69,7 @@ Create a service. The request JSON is a service object with details required for ### (TBD) List of services running in the cluster. ``` -GET /services +GET /ws/v1/services ``` #### Description @@ -108,7 +85,7 @@ Get a list of all currently running services (response includes a minimal projec ### Get current version of the API server. ``` -GET /services/version +GET /ws/v1/services/version ``` #### Description @@ -123,7 +100,7 @@ Get current version of the API server. ### Update a service or upgrade the binary version of the components of a running service ``` -PUT /services/{service_name} +PUT /ws/v1/services/{service_name} ``` #### Description @@ -147,7 +124,7 @@ Update the runtime properties of a service. Currently the following operations a ### Destroy a service ``` -DELETE /services/{service_name} +DELETE /ws/v1/services/{service_name} ``` #### Description @@ -170,7 +147,7 @@ Destroy a service and release all resources. This API might have to return JSON ### Get details of a service. ``` -GET /services/{service_name} +GET /ws/v1/services/{service_name} ``` #### Description @@ -193,7 +170,7 @@ Return the details (including containers) of a running service ### Flex a component's number of instances. ``` -PUT /services/{service_name}/components/{component_name} +PUT /ws/v1/services/{service_name}/components/{component_name} ``` #### Description