commit ae208a356248f399b059fdf86b4541f733b05998 Author: Eric Yang Date: Thu Sep 13 20:20:59 2018 -0400 YARN-8734. Implemented remote service dependency. Contributed by Eric Yang diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-api/src/main/resources/definition/YARN-Simplified-V1-API-Layer-For-Services.yaml b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-api/src/main/resources/definition/YARN-Simplified-V1-API-Layer-For-Services.yaml index d90ae06..d483b23 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-api/src/main/resources/definition/YARN-Simplified-V1-API-Layer-For-Services.yaml +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-api/src/main/resources/definition/YARN-Simplified-V1-API-Layer-For-Services.yaml @@ -198,6 +198,11 @@ definitions: required: - name - version + remote_service_dependencies: + type: array + items: + type: string + description: An array of services which should be in STABLE state, before this service can be started. properties: name: type: string diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/ServiceManager.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/ServiceManager.java index 04454b1..d922e63 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/ServiceManager.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/ServiceManager.java @@ -161,6 +161,8 @@ public State transition(ServiceManager serviceManager, serviceManager.serviceSpec.setState( ServiceState.UPGRADING); } + ServiceApiUtil.checkRemoteServiceDependencySatisified(serviceManager + .getServiceSpec()); return State.UPGRADING; } catch (Throwable e) { LOG.error("[SERVICE]: Upgrade to version {} failed", event.getVersion(), diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/ServiceMaster.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/ServiceMaster.java index 0caa119..60c2aed 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/ServiceMaster.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/ServiceMaster.java @@ -258,6 +258,12 @@ protected void loadApplicationJson(ServiceContext context, } @Override + public void start() { + ServiceApiUtil.checkRemoteServiceDependencySatisified(context.service); + super.start(); + } + + @Override protected void serviceStart() throws Exception { LOG.info("Starting service as user " + UserGroupInformation .getCurrentUser()); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/api/records/Service.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/api/records/Service.java index 22beff4..4acfb80 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/api/records/Service.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/api/records/Service.java @@ -74,6 +74,7 @@ private String version = null; private String description = null; private String dockerClientConfig = null; + private List remoteServiceDependencies = new ArrayList(); /** * A unique service name. @@ -353,6 +354,18 @@ public void setQueue(String queue) { this.queue = queue; } + @ApiModelProperty(example = "null", value = "A list of dependent services.") + @XmlElement(name = "remote_service_dependencies") + @JsonProperty("remote_service_dependencies") + public List getRemoteServiceDependencies() { + return remoteServiceDependencies; + } + + public void setRemoteServiceDependencies(List + remoteServiceDependencies) { + this.remoteServiceDependencies = remoteServiceDependencies; + } + public Service kerberosPrincipal(KerberosPrincipal kerberosPrincipal) { this.kerberosPrincipal = kerberosPrincipal; return this; @@ -438,6 +451,8 @@ public String toString() { .append(toIndentedString(kerberosPrincipal)).append("\n"); sb.append(" dockerClientConfig: ") .append(toIndentedString(dockerClientConfig)).append("\n"); + sb.append(" remote_service_dependencies: ") + .append(toIndentedString(remoteServiceDependencies)).append("\n"); sb.append("}"); return sb.toString(); } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/utils/ServiceApiUtil.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/utils/ServiceApiUtil.java index b588e88..3b28548 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/utils/ServiceApiUtil.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/utils/ServiceApiUtil.java @@ -35,6 +35,8 @@ import org.apache.hadoop.yarn.service.api.records.Container; import org.apache.hadoop.yarn.service.api.records.ContainerState; import org.apache.hadoop.yarn.service.api.records.Service; +import org.apache.hadoop.yarn.service.api.records.ServiceState; +import org.apache.hadoop.yarn.service.client.ServiceClient; import org.apache.hadoop.yarn.service.api.records.Artifact; import org.apache.hadoop.yarn.service.api.records.Component; import org.apache.hadoop.yarn.service.api.records.Configuration; @@ -695,4 +697,45 @@ private static String parseComponentName(String componentInstanceName) } return components; } + + private static boolean serviceDependencySatisfied(Service service) { + boolean result = true; + try { + List dependencies = service + .getRemoteServiceDependencies(); + org.apache.hadoop.conf.Configuration conf = + new org.apache.hadoop.conf.Configuration(); + if (dependencies != null && dependencies.size() > 0) { + ServiceClient sc = new ServiceClient(); + sc.init(conf); + sc.start(); + for (String dependent : dependencies) { + Service dependentService = sc.getStatus(dependent); + if (dependentService.getState() == null || + !dependentService.getState().equals(ServiceState.STABLE)) { + result = false; + LOG.info("Remote service dependency is not satisfied for " + + "service: {} state: {}", dependent, + dependentService.getState()); + } + } + sc.close(); + } + } catch (IOException | YarnException e) { + LOG.warn("Caught exception: ", e); + LOG.info("Remote service dependency is not satisified."); + result = false; + } + return result; + } + + public static void checkRemoteServiceDependencySatisified(Service service) { + while (!serviceDependencySatisfied(service)) { + try { + LOG.info("Waiting for remote service dependencies."); + Thread.sleep(15000L); + } catch (InterruptedException e) { + } + } + } } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/test/java/org/apache/hadoop/yarn/service/TestYarnNativeServices.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/test/java/org/apache/hadoop/yarn/service/TestYarnNativeServices.java index 216d88f..9b2be77 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/test/java/org/apache/hadoop/yarn/service/TestYarnNativeServices.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/test/java/org/apache/hadoop/yarn/service/TestYarnNativeServices.java @@ -437,6 +437,7 @@ public void testExpressUpgrade() throws Exception { // wait for upgrade to complete waitForServiceToBeStable(client, service); Service active = client.getStatus(service.getName()); + Assert.assertEquals("version mismatch", service.getVersion(), active.getVersion()); Assert.assertEquals("component not stable", ComponentState.STABLE, active.getComponent(component.getName()).getState()); Assert.assertEquals("compa does not have new env", "val1", diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/test/java/org/apache/hadoop/yarn/service/utils/TestServiceApiUtil.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/test/java/org/apache/hadoop/yarn/service/utils/TestServiceApiUtil.java index 98e7474..d252084 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/test/java/org/apache/hadoop/yarn/service/utils/TestServiceApiUtil.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/test/java/org/apache/hadoop/yarn/service/utils/TestServiceApiUtil.java @@ -30,6 +30,7 @@ import org.apache.hadoop.yarn.service.api.records.PlacementType; import org.apache.hadoop.yarn.service.api.records.Resource; import org.apache.hadoop.yarn.service.api.records.Service; +import org.apache.hadoop.yarn.service.api.records.ServiceState; import org.apache.hadoop.yarn.service.exceptions.RestApiErrorMessages; import org.junit.Assert; import org.junit.BeforeClass; @@ -733,6 +734,44 @@ public void testResolveNoCompsDependency() { } } + @Test(timeout = 1500) + public void testNoRemoteServiceDependencies() { + Service service = createExampleApplication(); + Component compa = createComponent("compa"); + Component compb = createComponent("compb"); + service.addComponent(compa); + service.addComponent(compb); + List remoteServiceDependencies = new ArrayList(); + service.setRemoteServiceDependencies(remoteServiceDependencies); + ServiceApiUtil.checkRemoteServiceDependencySatisified(service); + } + + @Test + public void testRemoteServiceDependencies() { + Thread thread = new Thread() { + @Override + public void run() { + Service service = createExampleApplication(); + Component compa = createComponent("compa"); + Component compb = createComponent("compb"); + service.addComponent(compa); + service.addComponent(compb); + List remoteServiceDependencies = new ArrayList(); + remoteServiceDependencies.add("abc"); + service.setRemoteServiceDependencies(remoteServiceDependencies); + Service dependent = createExampleApplication(); + dependent.setState(ServiceState.STOPPED); + ServiceApiUtil.checkRemoteServiceDependencySatisified(service); + } + }; + thread.start(); + try { + Thread.sleep(1000); + } catch (InterruptedException e) { + } + Assert.assertTrue(thread.isAlive()); + } + public static Service createExampleApplication() { Service exampleApp = new Service(); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/yarn-service/YarnServiceAPI.md b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/yarn-service/YarnServiceAPI.md index 4bfa742..c40c325 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/yarn-service/YarnServiceAPI.md +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/yarn-service/YarnServiceAPI.md @@ -406,7 +406,7 @@ a service resource has the following attributes. |queue|The YARN queue that this service should be submitted to.|false|string|| |kerberos_principal|The principal info of the user who launches the service|false|KerberosPrincipal|| |docker_client_config|URI of the file containing the docker client configuration (e.g. hdfs:///tmp/config.json)|false|string|| - +|remote_service_dependencies|A list of service names that this service depends on.| false | string array || ### ServiceState