diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/ServiceScheduler.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/ServiceScheduler.java index 7b809b9..ec5f3ed 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/ServiceScheduler.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/ServiceScheduler.java @@ -344,11 +344,7 @@ private void registerServiceInstance(ApplicationAttemptId attemptId, attemptId.getApplicationId().toString()); serviceRecord.set(YarnRegistryAttributes.YARN_PERSISTENCE, PersistencePolicies.APPLICATION); - serviceRecord.description = "Yarn Service Master"; - - serviceRecord.addExternalEndpoint(RegistryTypeUtils - .ipcEndpoint("classpath:org.apache.hadoop.yarn.service.appmaster.ipc", - context.clientAMService.getBindAddress())); + serviceRecord.description = "YarnServiceMaster"; // set any provided attributes setUserProvidedServiceRecordAttributes(service.getConfiguration(), diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/ServiceClient.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/ServiceClient.java index 6890aef..a3a9fd0 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/ServiceClient.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/ServiceClient.java @@ -170,6 +170,9 @@ private Service loadAppJsonFromLocalFS( if (!StringUtils.isEmpty(args.getServiceName())) { service.setName(args.getServiceName()); } + if (!StringUtils.isEmpty(args.queue)) { + service.setQueue(args.queue); + } return service; } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/params/AbstractClusterBuildingActionArgs.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/params/AbstractClusterBuildingActionArgs.java index 434c6d5..4ecbe9c 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/params/AbstractClusterBuildingActionArgs.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/params/AbstractClusterBuildingActionArgs.java @@ -37,7 +37,7 @@ public File getFile() { } @Parameter(names = { - ARG_QUEUE }, description = "Queue to submit the service") + ARG_QUEUE, ARG_SHORT_QUEUE}, description = "Queue to submit the service") public String queue; @Parameter(names = { diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/params/Arguments.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/params/Arguments.java index 9672621..67571e2 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/params/Arguments.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/params/Arguments.java @@ -77,6 +77,7 @@ String ARG_PATH = "--path"; String ARG_PRINCIPAL = "--principal"; String ARG_QUEUE = "--queue"; + String ARG_SHORT_QUEUE = "-q"; String ARG_LIFETIME = "--lifetime"; String ARG_RESOURCE = "--resource"; String ARG_RESOURCE_MANAGER = "--rm"; diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/component/instance/ComponentInstance.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/component/instance/ComponentInstance.java index 7d6525b..0e47ece 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/component/instance/ComponentInstance.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/component/instance/ComponentInstance.java @@ -360,7 +360,7 @@ private void updateServiceRecord( record.description = getCompInstanceName(); record.set(YarnRegistryAttributes.YARN_PERSISTENCE, PersistencePolicies.CONTAINER); - record.set("yarn:ip", status.getIPs()); + record.set("yarn:ip", status.getIPs().get(0)); record.set("yarn:hostname", status.getHost()); try { yarnRegistry diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-registry/src/main/java/org/apache/hadoop/registry/server/dns/ApplicationServiceRecordProcessor.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-registry/src/main/java/org/apache/hadoop/registry/server/dns/ApplicationServiceRecordProcessor.java index e6a1b5b..0b5f724 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-registry/src/main/java/org/apache/hadoop/registry/server/dns/ApplicationServiceRecordProcessor.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-registry/src/main/java/org/apache/hadoop/registry/server/dns/ApplicationServiceRecordProcessor.java @@ -18,6 +18,8 @@ import org.apache.hadoop.registry.client.types.Endpoint; import org.apache.hadoop.registry.client.types.ServiceRecord; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; import org.xbill.DNS.Name; import org.xbill.DNS.Type; @@ -32,7 +34,8 @@ */ public class ApplicationServiceRecordProcessor extends BaseServiceRecordProcessor { - + private static final Logger LOG = + LoggerFactory.getLogger(ApplicationServiceRecordProcessor.class); /** * Create an application service record processor. * @@ -57,6 +60,10 @@ public ApplicationServiceRecordProcessor( */ @Override public void initTypeToInfoMapping(ServiceRecord serviceRecord) throws Exception { + if (serviceRecord.external.isEmpty()) { + LOG.info(serviceRecord.description + ": No external endpoints defined."); + return; + } for (int type : getRecordTypes()) { switch (type) { case Type.A: @@ -309,6 +316,9 @@ public AApplicationRecordDescriptor(String path, throws Exception { this.setNames(new Name[] {getServiceName()}); List endpoints = serviceRecord.external; + if (endpoints.isEmpty()) { + return; + } // TODO: do we need a "hostname" attribute for an application record or // can we rely on the first endpoint record. this.setTarget(InetAddress.getByName( @@ -342,6 +352,9 @@ public AAAAApplicationRecordDescriptor(String path, @Override protected void init(ServiceRecord serviceRecord) throws Exception { super.init(serviceRecord); + if (getTarget() == null) { + return; + } try { this.setTarget(getIpv6Address(getTarget())); } catch (UnknownHostException e) { diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-registry/src/main/java/org/apache/hadoop/registry/server/dns/BaseServiceRecordProcessor.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-registry/src/main/java/org/apache/hadoop/registry/server/dns/BaseServiceRecordProcessor.java index 2fe3a6c..fd5c74f 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-registry/src/main/java/org/apache/hadoop/registry/server/dns/BaseServiceRecordProcessor.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-registry/src/main/java/org/apache/hadoop/registry/server/dns/BaseServiceRecordProcessor.java @@ -52,8 +52,8 @@ private String domain; private static final Pattern USER_NAME = Pattern.compile("/users/(\\w*)/?"); - private static final String SLIDER_API_PREFIX = - "classpath:org.apache.slider."; + private static final String YARN_SERVICE_API_PREFIX = + "classpath:org.apache.hadoop.yarn.service."; private static final String HTTP_API_TYPE = "http://"; /** @@ -425,8 +425,8 @@ protected int getPort(Endpoint endpoint) { */ protected String getDNSApiFragment(String api) { String dnsApi = null; - if (api.startsWith(SLIDER_API_PREFIX)) { - dnsApi = api.substring(SLIDER_API_PREFIX.length()); + if (api.startsWith(YARN_SERVICE_API_PREFIX)) { + dnsApi = api.substring(YARN_SERVICE_API_PREFIX.length()); } else if (api.startsWith(HTTP_API_TYPE)) { dnsApi = "http"; } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-registry/src/test/java/org/apache/hadoop/registry/server/dns/TestRegistryDNS.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-registry/src/test/java/org/apache/hadoop/registry/server/dns/TestRegistryDNS.java index cc839cc..65346e4 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-registry/src/test/java/org/apache/hadoop/registry/server/dns/TestRegistryDNS.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-registry/src/test/java/org/apache/hadoop/registry/server/dns/TestRegistryDNS.java @@ -69,7 +69,7 @@ + " \"type\" : \"JSONServiceRecord\",\n" + " \"description\" : \"Slider Application Master\",\n" + " \"external\" : [ {\n" - + " \"api\" : \"classpath:org.apache.slider.appmaster.ipc\",\n" + + " \"api\" : \"classpath:org.apache.hadoop.yarn.service.appmaster.ipc\",\n" + " \"addressType\" : \"host/port\",\n" + " \"protocolType\" : \"hadoop/IPC\",\n" + " \"addresses\" : [ {\n" @@ -84,7 +84,7 @@ + " \"uri\" : \"http://192.168.1.5:1027\"\n" + " } ]\n" + " }, {\n" - + " \"api\" : \"classpath:org.apache.slider.management\",\n" + + " \"api\" : \"classpath:org.apache.hadoop.yarn.service.management\",\n" + " \"addressType\" : \"uri\",\n" + " \"protocolType\" : \"REST\",\n" + " \"addresses\" : [ {\n" @@ -92,14 +92,14 @@ + " } ]\n" + " } ],\n" + " \"internal\" : [ {\n" - + " \"api\" : \"classpath:org.apache.slider.agents.secure\",\n" + + " \"api\" : \"classpath:org.apache.hadoop.yarn.service.agents.secure\",\n" + " \"addressType\" : \"uri\",\n" + " \"protocolType\" : \"REST\",\n" + " \"addresses\" : [ {\n" + " \"uri\" : \"https://192.168.1.5:47700/ws/v1/slider/agents\"\n" + " } ]\n" + " }, {\n" - + " \"api\" : \"classpath:org.apache.slider.agents.oneway\",\n" + + " \"api\" : \"classpath:org.apache.hadoop.yarn.service.agents.oneway\",\n" + " \"addressType\" : \"uri\",\n" + " \"protocolType\" : \"REST\",\n" + " \"addresses\" : [ {\n" diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/DockerLinuxContainerRuntime.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/DockerLinuxContainerRuntime.java index 89d2fc2..436d5b2 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/DockerLinuxContainerRuntime.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/DockerLinuxContainerRuntime.java @@ -628,13 +628,13 @@ public void reapContainer(ContainerRuntimeContext ctx) .executePrivilegedOperation(null, privOp, null, container.getLaunchContext().getEnvironment(), true, false); LOG.info("Docker inspect output for " + containerId + ": " + output); + // strip off quotes if any + output = output.replaceAll("['\"]", ""); int index = output.lastIndexOf(','); if (index == -1) { LOG.error("Incorrect format for ip and host"); return null; } - // strip off quotes if any - output = output.replaceAll("['\"]", ""); String ips = output.substring(0, index).trim(); String host = output.substring(index+1).trim(); String[] ipAndHost = new String[2]; diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/YarnCommands.md b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/YarnCommands.md index 55d007b..e464d54 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/YarnCommands.md +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/YarnCommands.md @@ -92,6 +92,7 @@ Usage `yarn service [sub-command] [service-name] [options]` Options: --file,-f The local path to the service definition file. + --queue,-q The queue to which the service is submitted. --example,-e The name of the example service such as: Sleeper A simple service that launches a few non-docker sleep containers on YARN. ```