commit 0076fd381b8b75dd15cbecb603dc2466df9ed245 Author: eyang Date: Thu Sep 28 11:21:09 2017 -0400 YARN-6626. Embed ApiServer into ResourceManager. (contributed by eyang) diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java index 114453f..af4107f 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java @@ -336,6 +336,8 @@ private static void addDeprecatedKeys() { public static final String YARN_WEBAPP_UI2_WARFILE_PATH = "yarn." + "webapp.ui2.war-file-path"; + public static final String YARN_API_SERVICES_ENABLE = "yarn." + + "webapp.api-service.enable"; public static final String RM_RESOURCE_TRACKER_ADDRESS = RM_PREFIX + "resource-tracker.address"; diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/main/java/org/apache/hadoop/yarn/service/webapp/ApiServer.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/main/java/org/apache/hadoop/yarn/service/webapp/ApiServer.java index e8286ef..3abd404 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/main/java/org/apache/hadoop/yarn/service/webapp/ApiServer.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/main/java/org/apache/hadoop/yarn/service/webapp/ApiServer.java @@ -17,6 +17,7 @@ package org.apache.hadoop.yarn.service.webapp; +import com.google.inject.Inject; import com.google.inject.Singleton; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.util.VersionInfo; @@ -24,6 +25,7 @@ import org.apache.hadoop.yarn.conf.YarnConfiguration; import org.apache.hadoop.yarn.exceptions.ApplicationNotFoundException; import org.apache.hadoop.yarn.exceptions.YarnException; +import org.apache.hadoop.yarn.server.webapp.WebServices; import org.apache.hadoop.yarn.service.api.records.Component; import org.apache.hadoop.yarn.service.api.records.Service; import org.apache.hadoop.yarn.service.api.records.ServiceState; @@ -57,6 +59,12 @@ @Singleton @Path(CONTEXT_ROOT) public class ApiServer { + + @Inject + public ApiServer(Configuration conf) { + super(); + } + private static final Logger LOG = LoggerFactory.getLogger(ApiServer.class); private static Configuration YARN_CONFIG = new YarnConfiguration(); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/pom.xml b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/pom.xml index fb07edc..205a64d 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/pom.xml +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/pom.xml @@ -245,12 +245,6 @@ - org.apache.hadoop - hadoop-yarn-server-resourcemanager - test - - - org.apache.curator curator-test test diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/api/records/Artifact.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/api/records/Artifact.java index 4328c6f..ce062cc 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/api/records/Artifact.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/api/records/Artifact.java @@ -23,6 +23,9 @@ import java.io.Serializable; import java.util.Objects; +import javax.xml.bind.annotation.XmlEnum; +import javax.xml.bind.annotation.XmlType; + import com.fasterxml.jackson.annotation.JsonInclude; import com.fasterxml.jackson.annotation.JsonProperty; import com.fasterxml.jackson.annotation.JsonValue; @@ -42,6 +45,11 @@ private String id = null; + /** + * Artifact Type. DOCKER, TARBALL or SERVICE + **/ + @XmlType(name = "artifact_type") + @XmlEnum public enum TypeEnum { DOCKER("DOCKER"), TARBALL("TARBALL"), SERVICE("SERVICE"); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/api/records/ConfigFile.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/api/records/ConfigFile.java index c1502c7..fb088e2 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/api/records/ConfigFile.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/api/records/ConfigFile.java @@ -26,7 +26,10 @@ import org.apache.hadoop.classification.InterfaceStability; import javax.xml.bind.annotation.XmlElement; +import javax.xml.bind.annotation.XmlEnum; import javax.xml.bind.annotation.XmlRootElement; +import javax.xml.bind.annotation.XmlType; + import java.io.Serializable; import java.util.HashMap; import java.util.Map; @@ -45,6 +48,11 @@ public class ConfigFile implements Serializable { private static final long serialVersionUID = -7009402089417704612L; + /** + * Config Type. XML, JSON, YAML, TEMPLATE, ENV and HADOOP_XML are supported. + **/ + @XmlType(name = "config_type") + @XmlEnum public enum TypeEnum { XML("XML"), PROPERTIES("PROPERTIES"), JSON("JSON"), YAML("YAML"), TEMPLATE( "TEMPLATE"), ENV("ENV"), HADOOP_XML("HADOOP_XML"),; diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/api/records/ReadinessCheck.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/api/records/ReadinessCheck.java index 1a25a4c..b25828f 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/api/records/ReadinessCheck.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/api/records/ReadinessCheck.java @@ -25,6 +25,9 @@ import java.util.Map; import java.util.Objects; +import javax.xml.bind.annotation.XmlEnum; +import javax.xml.bind.annotation.XmlType; + import com.fasterxml.jackson.annotation.JsonProperty; import com.fasterxml.jackson.annotation.JsonValue; import org.apache.hadoop.classification.InterfaceAudience; @@ -43,6 +46,11 @@ public class ReadinessCheck implements Serializable { private static final long serialVersionUID = -3836839816887186801L; + /** + * Type. HTTP and PORT + **/ + @XmlType(name = "type") + @XmlEnum public enum TypeEnum { HTTP("HTTP"), PORT("PORT"); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml index 0440458..960b8f0 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml @@ -212,6 +212,14 @@ + + Enable services rest api on ResourceManager. + + yarn.webapp.api-service.enable + true + + + yarn.resourcemanager.resource-tracker.address ${yarn.resourcemanager.hostname}:8031 diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMWebApp.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMWebApp.java index 4e36665..bee9354 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMWebApp.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMWebApp.java @@ -22,6 +22,8 @@ import java.net.InetSocketAddress; +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; import org.apache.hadoop.ha.HAServiceProtocol.HAServiceState; import org.apache.hadoop.yarn.conf.YarnConfiguration; import org.apache.hadoop.yarn.util.RMHAUtils; @@ -38,8 +40,12 @@ */ public class RMWebApp extends WebApp implements YarnWebParams { + private static final Log LOG = + LogFactory.getLog(RMWebApp.class.getName()); private final ResourceManager rm; private boolean standby = false; + private final static String APISERVER = + "org.apache.hadoop.yarn.service.webapp.ApiServer"; public RMWebApp(ResourceManager rm) { this.rm = rm; @@ -53,6 +59,19 @@ public void setup() { bind(RMWebApp.class).toInstance(this); if (rm != null) { + boolean enableServiceApi = rm.getConfig() + .getBoolean(YarnConfiguration.YARN_API_SERVICES_ENABLE, false); + if (enableServiceApi) { + try { + // Use reflection here to load ApiServer class, + // this is done to avoid creating cyclic dependency + // between maven projects. + Class apiServer = Class.forName(APISERVER); + bind(apiServer); + } catch (ClassNotFoundException e) { + LOG.warn("ApiServer REST API is not activated."); + } + } bind(ResourceManager.class).toInstance(rm); } route("/", RmController.class); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/yarn-service/QuickStart.md b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/yarn-service/QuickStart.md index ab415de..2b7a4e3 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/yarn-service/QuickStart.md +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/yarn-service/QuickStart.md @@ -101,17 +101,41 @@ yarn service destroy ${SERVICE_NAME} ``` ## Manage services on YARN via REST API -Below steps walk you through deploying services on YARN via REST API. - Refer to [API doc](YarnServiceAPI.md) for the detailed API specificatiosn. -### Start API-Server for deploying services on YARN + +YARN API Server REST API can be activated in two modes: embedded or standalone. + +### Start Embedded API-Server as part of ResourceManager +For running with ResourceManager, add this property to `yarn-site.xml` and restart ResourceManager. + +``` + + + Enable services rest api on ResourceManager. + + yarn.webapp.api-service.enable + true + +``` + +This configuration enables to deploy YARN service through ResourceManager web port. + +### Start Standalone API-Server for deploying services on YARN API server is the service that sits in front of YARN ResourceManager and lets users submit their API specs via HTTP. ``` yarn --daemon start apiserver ``` The above command starts the API Server on the localhost at port 9191 by default. +Refer to [API doc](YarnServiceAPI.md) for the detailed API specificatiosn. + ### Deploy a service -POST the aforementioned example service definition to the api-server endpoint: + +POST the aforementioned example service definition to the ResourceManager api-server endpoint: +``` +POST http://localhost:8088/ws/v1/services +``` + +Or standalone API server: ``` POST http://localhost:9191/ws/v1/services ``` @@ -222,4 +246,4 @@ If it is configured to use the standard privileged port `53`, the DNS server nee ``` sudo su - -c "yarn org.apache.hadoop.registry.server.dns.RegistryDNSServer > /${HADOOP_LOG_FOLDER}/registryDNS.log 2>&1 &" root ``` -Please refer to [YARN DNS doc](ServicesDiscovery.md) for the full list of configurations. \ No newline at end of file +Please refer to [YARN DNS doc](ServicesDiscovery.md) for the full list of configurations.