diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/main/resources/definition/YARN-Simplified-V1-API-Layer-For-Services.yaml b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/main/resources/definition/YARN-Simplified-V1-API-Layer-For-Services.yaml index cc76259..33da4c6 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/main/resources/definition/YARN-Simplified-V1-API-Layer-For-Services.yaml +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/main/resources/definition/YARN-Simplified-V1-API-Layer-For-Services.yaml @@ -233,6 +233,9 @@ definitions: configuration: description: Config properties of a service. Configurations provided at the service/global level are available to all the components. Specific properties can be overridden at the component level. $ref: '#/definitions/Configuration' + yarn_configuration: + description: Config properties for the service framework. Only the properties field in the Configuration object is used for the key/value pair of configs. This is different from the configuration filed in that the configuration field is used for service specific configs, whereas this is for the general yarn configs. + $ref: '#/definitions/Configuration' state: description: State of the service. Specifying a value for this attribute for the PUT payload means update the service to this desired state. $ref: '#/definitions/ServiceState' @@ -351,7 +354,7 @@ definitions: Configuration: description: Set of configuration properties that can be injected into the service components via envs, files and custom pluggable helper docker containers. Files of several standard formats like xml, properties, json, yaml and templates will be supported. properties: - properties: + props: type: object description: A blob of key-value pairs of common service properties. additionalProperties: diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/ServiceMaster.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/ServiceMaster.java index 2abdae1..a2c6320 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/ServiceMaster.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/ServiceMaster.java @@ -75,6 +75,11 @@ protected void serviceInit(Configuration conf) throws Exception { fs.setAppDir(appDir); loadApplicationJson(context, fs); + // Take yarn config from YarnFile and merge them into YarnConfiguration + for (Map.Entry entry : context.service + .getYarnConfiguration().getProps().entrySet()) { + conf.set(entry.getKey(), entry.getValue()); + } ContainerId amContainerId = getAMContainerId(); ApplicationAttemptId attemptId = amContainerId.getApplicationAttemptId(); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/ServiceScheduler.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/ServiceScheduler.java index 631f89e..4e683ab 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/ServiceScheduler.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/ServiceScheduler.java @@ -429,7 +429,7 @@ private void registerServiceInstance(ApplicationAttemptId attemptId, private void setUserProvidedServiceRecordAttributes( org.apache.hadoop.yarn.service.api.records.Configuration conf, ServiceRecord record) { String prefix = "service.record.attribute"; - for (Map.Entry entry : conf.getProperties().entrySet()) { + for (Map.Entry entry : conf.getProps().entrySet()) { if (entry.getKey().startsWith(prefix)) { String key = entry.getKey().substring(prefix.length() + 1); record.set(key, entry.getValue().trim()); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/api/ServiceApiConstants.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/api/ServiceApiConstants.java index a85191c..a4f2243 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/api/ServiceApiConstants.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/api/ServiceApiConstants.java @@ -53,9 +53,9 @@ String CONTAINER_ID = $("CONTAINER_ID"); // Templates for component instance host/IP - String COMPONENT_HOST = $("%s_HOST"); + String COMPONENT_INSTANCE_HOST = $("%s_HOST"); - String COMPONENT_IP = $("%s_IP"); + String COMPONENT_INSTANCE_IP = $("%s_IP"); // Constants for default cluster ZK String CLUSTER_ZK_QUORUM = $("CLUSTER_ZK_QUORUM"); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/api/records/Configuration.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/api/records/Configuration.java index 2f8ca96..239d57f 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/api/records/Configuration.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/api/records/Configuration.java @@ -41,32 +41,32 @@ **/ @InterfaceAudience.Public @InterfaceStability.Unstable -@ApiModel(description = "Set of configuration properties that can be injected into the service components via envs, files and custom pluggable helper docker containers. Files of several standard formats like xml, properties, json, yaml and templates will be supported.") +@ApiModel(description = "Set of configuration props that can be injected into the service components via envs, files and custom pluggable helper docker containers. Files of several standard formats like xml, props, json, yaml and templates will be supported.") @javax.annotation.Generated(value = "class io.swagger.codegen.languages.JavaClientCodegen", date = "2016-06-02T08:15:05.615-07:00") @JsonInclude(JsonInclude.Include.NON_NULL) public class Configuration implements Serializable { private static final long serialVersionUID = -4330788704981074466L; - private Map properties = new HashMap(); + private Map props = new HashMap(); private Map env = new HashMap(); private List files = new ArrayList(); /** * A blob of key-value pairs of common service properties. **/ - public Configuration properties(Map properties) { - this.properties = properties; + public Configuration props(Map props) { + this.props = props; return this; } - @ApiModelProperty(example = "null", value = "A blob of key-value pairs of common service properties.") - @JsonProperty("properties") - public Map getProperties() { - return properties; + @ApiModelProperty(example = "null", value = "A blob of key-value pairs of common service props.") + @JsonProperty("props") + public Map getProps() { + return props; } - public void setProperties(Map properties) { - this.properties = properties; + public void setProps(Map props) { + this.props = props; } /** @@ -141,11 +141,11 @@ public String getProperty(String name, String defaultValue) { } public void setProperty(String name, String value) { - properties.put(name, value); + props.put(name, value); } public String getProperty(String name) { - return properties.get(name.trim()); + return props.get(name.trim()); } public String getEnv(String name) { @@ -161,14 +161,14 @@ public boolean equals(java.lang.Object o) { return false; } Configuration configuration = (Configuration) o; - return Objects.equals(this.properties, configuration.properties) + return Objects.equals(this.props, configuration.props) && Objects.equals(this.env, configuration.env) && Objects.equals(this.files, configuration.files); } @Override public int hashCode() { - return Objects.hash(properties, env, files); + return Objects.hash(props, env, files); } @Override @@ -176,7 +176,7 @@ public String toString() { StringBuilder sb = new StringBuilder(); sb.append("class Configuration {\n"); - sb.append(" properties: ").append(toIndentedString(properties)) + sb.append(" props: ").append(toIndentedString(props)) .append("\n"); sb.append(" env: ").append(toIndentedString(env)).append("\n"); sb.append(" files: ").append(toIndentedString(files)).append("\n"); @@ -201,8 +201,8 @@ private String toIndentedString(java.lang.Object o) { * this ConfigFile. */ public synchronized void mergeFrom(Configuration that) { - SliderUtils.mergeMapsIgnoreDuplicateKeys(this.properties, that - .getProperties()); + SliderUtils.mergeMapsIgnoreDuplicateKeys(this.props, that + .getProps()); SliderUtils.mergeMapsIgnoreDuplicateKeys(this.env, that.getEnv()); Map thatMap = new HashMap<>(); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/api/records/Service.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/api/records/Service.java index 77a2610..fc6853a 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/api/records/Service.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/api/records/Service.java @@ -58,6 +58,7 @@ private Long lifetime = null; private PlacementPolicy placementPolicy = null; private List components = new ArrayList<>(); + private Configuration yarnConfiguration = new Configuration(); private Configuration configuration = new Configuration(); private ServiceState state = null; private Map quicklinks = new HashMap<>(); @@ -257,10 +258,23 @@ public Component getComponent(String name) { } /** - * Config properties of an service. Configurations provided at the - * service/global level are available to all the components. Specific - * properties can be overridden at the component level. + * Config properties for YARN **/ + public Service yarnConfiguration(Configuration yarnConfiguration) { + this.yarnConfiguration = yarnConfiguration; + return this; + } + + @ApiModelProperty(example = "null", value = "Config properties for YARN") + @JsonProperty("configuration") + public Configuration getYarnConfiguration() { + return yarnConfiguration; + } + + public void setYarnConfiguration(Configuration yarnConfiguration) { + this.yarnConfiguration = yarnConfiguration; + } + public Service configuration(Configuration configuration) { this.configuration = configuration; return this; @@ -369,6 +383,8 @@ public String toString() { .append("\n"); sb.append(" configuration: ").append(toIndentedString(configuration)) .append("\n"); + sb.append(" yarnConfiguration: ").append(toIndentedString(yarnConfiguration)) + .append("\n"); sb.append(" state: ").append(toIndentedString(state)).append("\n"); sb.append(" quicklinks: ").append(toIndentedString(quicklinks)) .append("\n"); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/ServiceClient.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/ServiceClient.java index f6d35e4..aaaca2f 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/ServiceClient.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/ServiceClient.java @@ -567,8 +567,9 @@ private ApplicationId submitApp(Service app) appTimeout.put(ApplicationTimeoutType.LIFETIME, app.getLifetime()); submissionContext.setApplicationTimeouts(appTimeout); } - submissionContext.setMaxAppAttempts(conf.getInt( - YarnServiceConf.AM_RESTART_MAX, 2)); + submissionContext.setMaxAppAttempts(YarnServiceConf + .getInt(YarnServiceConf.AM_RESTART_MAX, 20, app.getYarnConfiguration(), + conf)); Map localResources = new HashMap<>(); @@ -582,14 +583,14 @@ private ApplicationId submitApp(Service app) if (LOG.isDebugEnabled()) { printLocalResources(localResources); } - Map env = addAMEnv(conf); + Map env = addAMEnv(); // create AM CLI - String cmdStr = - buildCommandLine(serviceName, conf, appRootDir, hasAMLog4j); + String cmdStr = buildCommandLine(serviceName, conf, appRootDir, hasAMLog4j); submissionContext.setResource(Resource.newInstance(YarnServiceConf - .getLong(YarnServiceConf.AM_RESOURCE_MEM, YarnServiceConf.DEFAULT_KEY_AM_RESOURCE_MEM, - app.getConfiguration(), conf), 1)); + .getLong(YarnServiceConf.AM_RESOURCE_MEM, + YarnServiceConf.DEFAULT_KEY_AM_RESOURCE_MEM, + app.getYarnConfiguration(), conf), 1)); String queue = app.getQueue(); if (StringUtils.isEmpty(queue)) { queue = conf.get(YARN_QUEUE, "default"); @@ -649,7 +650,7 @@ private String buildCommandLine(String serviceName, Configuration conf, return cmdStr; } - private Map addAMEnv(Configuration conf) throws IOException { + private Map addAMEnv() throws IOException { Map env = new HashMap<>(); ClasspathConstructor classpath = buildClasspath(YarnServiceConstants.SUBMITTED_CONF_DIR, "lib", fs, getConfig() diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/component/Component.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/component/Component.java index a5dd39c..7078615 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/component/Component.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/component/Component.java @@ -468,9 +468,9 @@ public boolean areDependenciesReady() { } String ip = instance.getContainerStatus().getIPs().get(0); String host = instance.getContainerStatus().getHost(); - tokens.put(String.format(COMPONENT_IP, + tokens.put(String.format(COMPONENT_INSTANCE_IP, instance.getCompInstanceName().toUpperCase()), ip); - tokens.put(String.format(COMPONENT_HOST, + tokens.put(String.format(COMPONENT_INSTANCE_HOST, instance.getCompInstanceName().toUpperCase()), host); } } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/conf/YarnServiceConf.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/conf/YarnServiceConf.java index 1968e95..708e605 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/conf/YarnServiceConf.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/conf/YarnServiceConf.java @@ -28,7 +28,7 @@ // Retry settings for container failures public static final String CONTAINER_RETRY_MAX = "yarn.service.container-failure.retry.max"; - public static final String CONTAINER_RETRY_INTERVAL = "yarn.service.container-failure.retry-interval"; + public static final String CONTAINER_RETRY_INTERVAL = "yarn.service.container-failure.retry-interval-ms"; public static final String AM_RESTART_MAX = "yarn.service.am-restart.max-attempts"; public static final String AM_RESOURCE_MEM = "yarn.service.am-resource.memory"; diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/timelineservice/ServiceTimelinePublisher.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/timelineservice/ServiceTimelinePublisher.java index cfe7c1b..e254d77 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/timelineservice/ServiceTimelinePublisher.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/timelineservice/ServiceTimelinePublisher.java @@ -274,7 +274,7 @@ private void publishComponents(List components) { private void publishUserConf(Configuration configuration, String entityId, String entityType) { - populateTimelineEntity(configuration.getProperties().entrySet().iterator(), + populateTimelineEntity(configuration.getProps().entrySet().iterator(), entityId, entityType); populateTimelineEntity(configuration.getEnv().entrySet().iterator(), diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/test/java/org/apache/hadoop/yarn/service/conf/TestAppJsonResolve.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/test/java/org/apache/hadoop/yarn/service/conf/TestAppJsonResolve.java index 8739382..8afe481 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/test/java/org/apache/hadoop/yarn/service/conf/TestAppJsonResolve.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/test/java/org/apache/hadoop/yarn/service/conf/TestAppJsonResolve.java @@ -56,7 +56,7 @@ public void testOverride() throws Throwable { assertEquals(2, global.getFiles().size()); Configuration simple = orig.getComponent("simple").getConfiguration(); - assertEquals(0, simple.getProperties().size()); + assertEquals(0, simple.getProps().size()); assertEquals(1, simple.getFiles().size()); Configuration master = orig.getComponent("master").getConfiguration(); @@ -66,7 +66,7 @@ public void testOverride() throws Throwable { Configuration worker = orig.getComponent("worker").getConfiguration(); LOG.info("worker = {}", worker); - assertEquals(3, worker.getProperties().size()); + assertEquals(3, worker.getProps().size()); assertEquals(0, worker.getFiles().size()); assertEquals("worker", worker.getProperty("name")); @@ -86,7 +86,7 @@ public void testOverride() throws Throwable { assertEquals(2, global.getFiles().size()); simple = orig.getComponent("simple").getConfiguration(); - assertEquals(2, simple.getProperties().size()); + assertEquals(2, simple.getProps().size()); assertEquals("a", simple.getProperty("g1")); assertEquals("b", simple.getProperty("g2")); assertEquals(2, simple.getFiles().size()); @@ -104,7 +104,7 @@ public void testOverride() throws Throwable { master = orig.getComponent("master").getConfiguration(); LOG.info("master = {}", master); - assertEquals(3, master.getProperties().size()); + assertEquals(3, master.getProps().size()); assertEquals("m", master.getProperty("name")); assertEquals("overridden", master.getProperty("g1")); assertEquals("b", master.getProperty("g2")); @@ -122,7 +122,7 @@ public void testOverride() throws Throwable { worker = orig.getComponent("worker").getConfiguration(); LOG.info("worker = {}", worker); - assertEquals(4, worker.getProperties().size()); + assertEquals(4, worker.getProps().size()); assertEquals("worker", worker.getProperty("name")); assertEquals("overridden-by-worker", worker.getProperty("g1")); @@ -139,19 +139,19 @@ public void testOverrideExternalConfiguration() throws IOException { Service orig = ExampleAppJson.loadResource(EXTERNAL_JSON_1); Configuration global = orig.getConfiguration(); - assertEquals(0, global.getProperties().size()); + assertEquals(0, global.getProps().size()); assertEquals(3, orig.getComponents().size()); Configuration simple = orig.getComponent("simple").getConfiguration(); - assertEquals(0, simple.getProperties().size()); + assertEquals(0, simple.getProps().size()); Configuration master = orig.getComponent("master").getConfiguration(); - assertEquals(1, master.getProperties().size()); + assertEquals(1, master.getProps().size()); assertEquals("is-overridden", master.getProperty("g3")); Configuration other = orig.getComponent("other").getConfiguration(); - assertEquals(0, other.getProperties().size()); + assertEquals(0, other.getProps().size()); // load the external service SliderFileSystem sfs = ServiceTestUtils.initMockFs(); @@ -165,19 +165,19 @@ public void testOverrideExternalConfiguration() throws IOException { YarnConfiguration()); global = orig.getConfiguration(); - assertEquals(0, global.getProperties().size()); + assertEquals(0, global.getProps().size()); assertEquals(4, orig.getComponents().size()); simple = orig.getComponent("simple").getConfiguration(); - assertEquals(3, simple.getProperties().size()); + assertEquals(3, simple.getProps().size()); assertEquals("a", simple.getProperty("g1")); assertEquals("b", simple.getProperty("g2")); assertEquals("60", simple.getProperty("yarn.service.failure-count-reset.window")); master = orig.getComponent("master").getConfiguration(); - assertEquals(5, master.getProperties().size()); + assertEquals(5, master.getProps().size()); assertEquals("512M", master.getProperty("jvm.heapsize")); assertEquals("overridden", master.getProperty("g1")); assertEquals("b", master.getProperty("g2")); @@ -187,7 +187,7 @@ public void testOverrideExternalConfiguration() throws IOException { Configuration worker = orig.getComponent("worker").getConfiguration(); LOG.info("worker = {}", worker); - assertEquals(4, worker.getProperties().size()); + assertEquals(4, worker.getProps().size()); assertEquals("512M", worker.getProperty("jvm.heapsize")); assertEquals("overridden-by-worker", worker.getProperty("g1")); assertEquals("b", worker.getProperty("g2")); @@ -195,6 +195,6 @@ public void testOverrideExternalConfiguration() throws IOException { worker.getProperty("yarn.service.failure-count-reset.window")); other = orig.getComponent("other").getConfiguration(); - assertEquals(0, other.getProperties().size()); + assertEquals(0, other.getProps().size()); } } \ No newline at end of file diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/yarn-service/Configurations.md b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/yarn-service/Configurations.md new file mode 100644 index 0000000..2a22af8 --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/yarn-service/Configurations.md @@ -0,0 +1,153 @@ + + +This document describes how to configure the services to be deployed on YARN + +There are mainly three types of configurations: + +* The configurations specific to the custom service running on YARN . E.g. the hbase-site.xml for a Hbase service running on YARN. + * It can be specified at both global service level(`Service#configuration`) or component level(`Component#configuration`). + * Service-level configs are considered as the default configs for all components and component-level config can override service level config. + * All config properties that uses constant variables as described below are subject to substitutions. +* The configurations specific to YARN. (`Service#yarn_configuration`). + * E.g. The `yarn.service.am-restart.max-attempts` which controls how many times +the framework AM can be retried if it fails. These configs are mainly to control the behavior of the framework AM , rather than the custom services launched by the framework. +* Some constants such as `COMPONENT_INSTANCE_NAME` for referring some system defined properties. + * They are substituted by the framework before writing the config files. + +Below describes the details for each type of configurations. + +## Configuration for custom service +Below is how a configuration object is typically structured +``` +"configuration" : { + "env" : { // The environment variables to be exported when container gets launched + "env1" : "val1" + }, + "files" : [ // The list of configuration files to be mounted for the container + { + "type": "HADOOP_XML", // The format of the config file into which the "props" are dumped + "dest_file": "/etc/hadoop/conf/core-site.xml", // The location where the config file is mounted inside the container + "props" : { // The list of key/value pairs to be dumped into the config file + "fs.defaultFS" : "hdfs://myhdfs" // This property will be written into core-site.xml + } + }, + { + "type": "HADOOP_XML", // The format of the config file. + "src_file" : ""hdfs://mycluster/tmp/conf/yarn-site.xml"" // The location of the source config file to be downloaded + "dest_file": "/etc/hadoop/conf/yarn-site.xml", // The location where the config file will be mounted inside the container/ + "props" : { + "yarn.resourcemanager.webapp.address" : "${COMPONENT_INSTANCE_NAME}.${SERVICE_NAME}.root.hwxdev.site" // Merge into yarn-site.xml or override the this property in yarn-site.xml + } + } + ] +} +``` + +* env : the environment variables to be exported when container gets launched. +* files : The list of configuration files to be mounted inside the container. + * type: The format of the config file(`dest_file`) to be mounted inside the container. If `src_file` is specified, it is also the format for both `src_file` and `dest_file`. + * HADOOP_XML : the standard hadoop xml format + * JSON : the standard JSON format + * YAML : the YAML format + * TEMPLATE : the plain text format. If this is used, use `content` as the key in `props` field, and the value will be the actual content to be written in the config file. E.g + + ``` + { + "type": "TEMPLATE" + "dest_file": "/etc/conf/hello" + "props" : { + "content" : "Hello world" + } + } + ``` + The string `Hello world` will be written into a file located at `/etc/conf/hello` + * src_file : [optional], the source location of the config file at a network accessible location such as hdfs. + * The format of both `src_file` and `dest_file` are defined by `type`. + * The `src_file` will be downloaded by YARN NodeManager and be mounted inside the container as in the location specified by `dest_file`. + * If any properties specified in the `props` field, they are added (or overwriting existing properties) in the `src_file`. + * If `src_file` is not specified, only the properties in the `props` field will be written into the `dest_file` file. + * dest_file : the location where the config file is mounted inside the container. The format is defined by `type`. + * properties : The list of key/value pair configs to be written into the `dest_file` in the format as defined in `type`. If `src_file` is specified, these properties will be added (or overwriting existing properties) in the `src_file`. + +## Configuration for YARN +This section describes the configurations for configuring the YARN service framework. + +These can be specified either in the cluster `yarn-site.xml` at the gloabl level or in the `yarn_configuration` field of the `service` object as per service basis like below: +``` +{ + "yarn_configuration" : { + "props" : { + "yarn.service.am-restart.max-attempts" : 10 + } + } +} +``` +Above config make the service AM to be retried at max 10 times. + +#### Available configurations: + +| Name | Description | +| ------------ | ------------- | +|yarn.service.client-am.retry.max-wait-ms | the max retry time in milliseconds for the service client to talk to the service AM. By default, it is set to 0, which means no retry | +|yarn.service.client-am.retry-interval-ms | the retry interval in milliseconds for the service client to talk to the service AM. By default, it is 2000, i.e. 2 seconds | +|yarn.service.container-failure.retry.max | the max number of retries for the container to be auto restarted if it fails. By default, it is set to -1, which means forever. +|yarn.service.container-failure.retry-interval-ms| the retry interval in milliseconds for the container to be restarted. By default, it is 30000, i.e. 30 seconds | +|yarn.service.am-restart.max-attempts| the max number of attempts for the framework AM +|yarn.service.am-resource.memory | the memory size in GB for the framework AM. By default, it is set to 1024 +|yarn.service.queue | the default queue to which the service will be submitted. By default, it is submitted to `default` queue +|yarn.service.base.path | the root location for the service artifacts on hdfs for a user. By default, it is under ${user_home_dir}/.yarn/ +|yarn.service.container-failure-per-component.threshold | the max number of container failures for a given component before the AM exits. +|yarn.service.node-blacklist.threshold | Maximum number of container failures on a node before the node is blacklisted by the AM +|yarn.service.failure-count-reset.window | The interval in seconds when the `yarn.service.container-failure-per-component.threshold` and `yarn.service.node-blacklist.threshold` gets reset. By default, it is 21600, i.e. 6 hours +|yarn.service.readiness-check-interval.seconds | The interval in seconds between readiness checks. By default, it is 30 seconds + +## Constant variables for service +The service framework provides some constant variables for user to configure their services. These variables are either dynamically generated by the system or are static ones which indicates some system properties such as service name. +User can use these constants in their configurations to be dynamically substituted by the system.E.g. +``` +{ + "type" : "HADOOP_XML", + "dest_file" : "/etc/hadoop/hbase-site.xml", + "props" : { + "hbase.regionserver.hostname": "${COMPONENT_INSTANCE_NAME}.${SERVICE_NAME}.root.hwxdev.site" + } +} +``` +Here, `COMPONENT_INSTANCE_NAME` and `SERVICE_NAME` are the constants to be substituted by the system. + +Suppose the `COMPONENT_INSTANCE_NAME` is `regionserver-0` and `SERVICE_NAME` is defined by user as `hbase`, +then, the config will be substituted by the system and written in the config file `/etc/hadoop/hbase-site.xml` inside the container as below: +``` + + hbase.regionserver.hostname + regionserver-0.hbase.root.hwxdev.site + +``` +where `regionserver-0` is the actual component instance name assigned by the system for this container and `hbase` is the service name defined by the user + +Below describes a set of constants that are available currently. + +| Name | Description | +| ------------ | ------------- | +| SERVICE_NAME | name of the service defined by the user +| USER | user who submits the service | +| DOMAIN | the domain name for the cluster | +| COMPONENT_NAME | the name for a given component | +| COMPONENT_INSTANCE_NAME | the name for a given component instance (i.e. container) | +| COMPONENT_ID | the monotonically increasing integer for a given component +| CONTAINER_ID | the YARN container Id for a given container | +| ${component_instance_name}_HOST | the hostname for a component instance (i.e. container), e.g. regionserver-0_HOST will be substituted by the actual hostname of the component instance | +| ${component_instance_name}_IP | the ip for a component instance (i.e. container), e.g. regionserver-0_IP will be substituted by the actual IP address of the component instance | +| CLUSTER_FS_URI | the URI of the cluster hdfs | diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/yarn-service/Overview.md b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/yarn-service/Overview.md index 58daee5..72c2d3e 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/yarn-service/Overview.md +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/yarn-service/Overview.md @@ -52,9 +52,10 @@ The benefits of combining these workloads are two-fold: * [Concepts](Concepts.md): Describes the internals of the framework and some features in YARN core to support running services on YARN. * [Service REST API](YarnServiceAPI.md): The API doc for deploying/managing services on YARN. -* [Service Discovery](ServiceDiscovery.md): Descirbes the service discovery mechanism on YARN. +* [Service Discovery](ServiceDiscovery.md): Describes the service discovery mechanism on YARN. * [Registry DNS](RegistryDNS.md): Deep dives into the Registry DNS internals. * [Examples](Examples.md): List some example service definitions (`Yarnfile`). +* [Configurations](Configurations.md): Describes how to configure the custom services on YARN. diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/yarn-service/YarnServiceAPI.md b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/yarn-service/YarnServiceAPI.md index e9aaf7e..187338f 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/yarn-service/YarnServiceAPI.md +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/yarn-service/YarnServiceAPI.md @@ -260,7 +260,7 @@ Set of configuration properties that can be injected into the service components |Name|Description|Required|Schema|Default| |----|----|----|----|----| -|properties|A blob of key-value pairs of common service properties.|false|object|| +|props|A blob of key-value pairs of common service properties.|false|object|| |env|A blob of key-value pairs which will be appended to the default system properties and handed off to the service at start time. All placeholder references to properties will be substituted before injection.|false|object|| |files|Array of list of files that needs to be created and made available as volumes in the service component containers.|false|ConfigFile array|| @@ -412,7 +412,7 @@ Note, lifetime value of -1 means unlimited lifetime. "memory": "256" }, "configuration": { - "properties": {}, + "props": {}, "env": {}, "files": [] }, @@ -443,7 +443,7 @@ Note, lifetime value of -1 means unlimited lifetime. } ], "configuration": { - "properties": {}, + "props": {}, "env": {}, "files": [] },