commit 79941dc2214ef8bfb976f648d5e93220026d270b Author: Eric Yang Date: Thu Sep 13 20:20:59 2018 -0400 YARN-8734. Implemented remote service dependency. Contributed by Eric Yang diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-api/src/main/resources/definition/YARN-Simplified-V1-API-Layer-For-Services.yaml b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-api/src/main/resources/definition/YARN-Simplified-V1-API-Layer-For-Services.yaml index d90ae06..d483b23 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-api/src/main/resources/definition/YARN-Simplified-V1-API-Layer-For-Services.yaml +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-api/src/main/resources/definition/YARN-Simplified-V1-API-Layer-For-Services.yaml @@ -198,6 +198,11 @@ definitions: required: - name - version + remote_service_dependencies: + type: array + items: + type: string + description: An array of services which should be in STABLE state, before this service can be started. properties: name: type: string diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/ServiceMaster.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/ServiceMaster.java index 0caa119..0901f0e 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/ServiceMaster.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/ServiceMaster.java @@ -41,11 +41,15 @@ import org.apache.hadoop.yarn.api.ApplicationConstants; import org.apache.hadoop.yarn.api.records.ApplicationAttemptId; import org.apache.hadoop.yarn.api.records.ContainerId; +import org.apache.hadoop.yarn.client.api.YarnClient; import org.apache.hadoop.yarn.conf.YarnConfiguration; import org.apache.hadoop.yarn.exceptions.YarnException; import org.apache.hadoop.yarn.security.AMRMTokenIdentifier; import org.apache.hadoop.yarn.security.client.ClientToAMTokenSecretManager; +import org.apache.hadoop.yarn.service.ServiceManager.State; +import org.apache.hadoop.yarn.service.api.records.Service; import org.apache.hadoop.yarn.service.api.records.ServiceState; +import org.apache.hadoop.yarn.service.client.ServiceClient; import org.apache.hadoop.yarn.service.exceptions.BadClusterStateException; import org.apache.hadoop.yarn.service.monitor.ServiceMonitor; import org.apache.hadoop.yarn.service.utils.ServiceApiUtil; @@ -60,7 +64,9 @@ import java.net.URISyntaxException; import java.nio.ByteBuffer; import java.security.PrivilegedExceptionAction; +import java.util.ArrayList; import java.util.Iterator; +import java.util.List; import java.util.Map; import static org.apache.hadoop.yarn.service.conf.YarnServiceConstants.KEYTAB_LOCATION; @@ -80,6 +86,7 @@ private String serviceKeytab; private String servicePrincipalName; protected ServiceContext context; + protected YarnClient yarnClient; public ServiceMaster(String name) { super(name); @@ -88,6 +95,7 @@ public ServiceMaster(String name) { @Override protected void serviceInit(Configuration conf) throws Exception { printSystemEnv(); + yarnClient = YarnClient.createYarnClient(); context = new ServiceContext(); Path appDir = getAppDir(); context.serviceHdfsDir = appDir.toString(); @@ -257,6 +265,47 @@ protected void loadApplicationJson(ServiceContext context, LOG.info(context.service.toString()); } + private boolean serviceDependencySatisfied() { + boolean result = true; + try { + List dependencies = context.service + .getRemoteServiceDependencies(); + Configuration conf = new Configuration(); + ServiceClient sc = new ServiceClient(); + sc.init(conf); + sc.start(); + if (dependencies != null && dependencies.size() > 0) { + for (String dependent : dependencies) { + Service dependentService = sc.getStatus(dependent); + if (dependentService.getState() == null || + !dependentService.getState().equals(ServiceState.STABLE)) { + result = false; + LOG.info("Remote service dependency is not satisfied for service: {}" + + " state: {}", dependent, dependentService.getState()); + } + } + } + sc.close(); + } catch (IOException | YarnException e) { + LOG.warn("Caught exception: ", e); + LOG.info("Remote service dependency is not satisified."); + result = false; + } + return result; + } + + @Override + public void start() { + while (!serviceDependencySatisfied()) { + try { + LOG.info("Waiting for remote service dependencies."); + Thread.sleep(15000L); + } catch (InterruptedException e) { + } + } + super.start(); + } + @Override protected void serviceStart() throws Exception { LOG.info("Starting service as user " + UserGroupInformation diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/api/records/Service.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/api/records/Service.java index 22beff4..bc3537b 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/api/records/Service.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/api/records/Service.java @@ -74,6 +74,7 @@ private String version = null; private String description = null; private String dockerClientConfig = null; + private List remoteServiceDependencies = new ArrayList(); /** * A unique service name. @@ -353,6 +354,17 @@ public void setQueue(String queue) { this.queue = queue; } + @ApiModelProperty(example = "null", required = false, value = "A list of dependent services.") + @XmlElement(name = "remote_service_dependencies") + @JsonProperty("remote_service_dependencies") + public List getRemoteServiceDependencies() { + return remoteServiceDependencies; + } + + public void setRemoteServiceDependencies(List remoteServiceDependencies) { + this.remoteServiceDependencies = remoteServiceDependencies; + } + public Service kerberosPrincipal(KerberosPrincipal kerberosPrincipal) { this.kerberosPrincipal = kerberosPrincipal; return this; @@ -438,6 +450,8 @@ public String toString() { .append(toIndentedString(kerberosPrincipal)).append("\n"); sb.append(" dockerClientConfig: ") .append(toIndentedString(dockerClientConfig)).append("\n"); + sb.append(" remote_service_dependencies: ") + .append(toIndentedString(remoteServiceDependencies)).append("\n"); sb.append("}"); return sb.toString(); }