diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/core/launch/AbstractLauncher.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/core/launch/AbstractLauncher.java index 965ea35..8f61bf4 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/core/launch/AbstractLauncher.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/core/launch/AbstractLauncher.java @@ -81,6 +81,7 @@ protected boolean yarnDockerMode = false; protected String dockerImage; protected String dockerNetwork = DEFAULT_DOCKER_NETWORK; + protected String dockerHostname; protected String yarnContainerMountPoints; protected String runPrivilegedContainer; @@ -236,6 +237,8 @@ public ContainerLaunchContext completeContainerLaunch() throws IOException { env.put("YARN_CONTAINER_RUNTIME_TYPE", "docker"); env.put("YARN_CONTAINER_RUNTIME_DOCKER_IMAGE", dockerImage); env.put("YARN_CONTAINER_RUNTIME_DOCKER_CONTAINER_NETWORK", dockerNetwork); + env.put("YARN_CONTAINER_RUNTIME_DOCKER_CONTAINER_HOSTNAME", + dockerHostname); env.put("YARN_CONTAINER_RUNTIME_DOCKER_RUN_PRIVILEGED_CONTAINER", runPrivilegedContainer); StringBuilder sb = new StringBuilder(); for (Entry mount : mountPaths.entrySet()) { @@ -469,6 +472,10 @@ public void setDockerNetwork(String dockerNetwork) { this.dockerNetwork = dockerNetwork; } + public void setDockerHostname(String dockerHostname) { + this.dockerHostname = dockerHostname; + } + public void setYarnContainerMountPoints(String yarnContainerMountPoints) { this.yarnContainerMountPoints = yarnContainerMountPoints; } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/AbstractProviderService.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/AbstractProviderService.java index b7fa802..e0d9402 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/AbstractProviderService.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/AbstractProviderService.java @@ -60,8 +60,9 @@ protected AbstractProviderService(String name) { super(name); } - public abstract void processArtifact(ContainerLauncher launcher, Component - component, SliderFileSystem fileSystem) throws IOException; + public abstract void processArtifact(ContainerLauncher launcher, + Application application, RoleInstance roleInstance, + SliderFileSystem fileSystem) throws IOException; @Override public void setAMState(StateAccessForProviders stateAccessor) { @@ -78,7 +79,7 @@ public void buildContainerLaunchContext(ContainerLauncher launcher, SliderFileSystem fileSystem, RoleInstance roleInstance) throws IOException, SliderException { Component component = providerRole.component; - processArtifact(launcher, component, fileSystem); + processArtifact(launcher, application, roleInstance, fileSystem); // Generate tokens (key-value pair) for config substitution. // Get pre-defined tokens diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/DefaultProviderService.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/DefaultProviderService.java index 7f7d209..8d2725c 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/DefaultProviderService.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/DefaultProviderService.java @@ -17,10 +17,10 @@ */ package org.apache.slider.providers; -import org.apache.slider.api.resource.Component; +import org.apache.slider.api.resource.Application; import org.apache.slider.common.tools.SliderFileSystem; import org.apache.slider.core.launch.ContainerLauncher; -import org.apache.slider.providers.AbstractProviderService; +import org.apache.slider.server.appmaster.state.RoleInstance; import java.io.IOException; @@ -31,7 +31,8 @@ protected DefaultProviderService() { } @Override - public void processArtifact(ContainerLauncher launcher, Component - component, SliderFileSystem fileSystem) throws IOException { + public void processArtifact(ContainerLauncher launcher, Application + application, RoleInstance roleInstance, SliderFileSystem fileSystem) + throws IOException { } } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/ProviderUtils.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/ProviderUtils.java index 0da535e..ecc521f 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/ProviderUtils.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/ProviderUtils.java @@ -504,8 +504,7 @@ public void updateServiceRecord(StateAccessForProviders amState, // create and publish updated service record (including hostname & ip) ServiceRecord record = new ServiceRecord(); record.set(YarnRegistryAttributes.YARN_ID, containerId); - String componentInstanceName = role.getCompInstanceName(); - record.description = componentInstanceName.replaceAll("_", "-"); + record.description = role.getCompInstanceName(); record.set(YarnRegistryAttributes.YARN_PERSISTENCE, PersistencePolicies.CONTAINER); // TODO: use constants from YarnRegistryAttributes diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/docker/DockerProviderService.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/docker/DockerProviderService.java index f35d4d1..73783af 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/docker/DockerProviderService.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/docker/DockerProviderService.java @@ -17,14 +17,19 @@ */ package org.apache.slider.providers.docker; +import org.apache.hadoop.registry.client.api.RegistryConstants; +import org.apache.hadoop.registry.client.binding.RegistryUtils; +import org.apache.slider.api.resource.Application; import org.apache.slider.api.resource.Component; import org.apache.slider.common.tools.SliderFileSystem; import org.apache.slider.core.launch.ContainerLauncher; import org.apache.slider.providers.AbstractProviderService; +import org.apache.slider.server.appmaster.state.RoleInstance; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import java.io.IOException; +import java.text.MessageFormat; public class DockerProviderService extends AbstractProviderService implements DockerKeys { @@ -36,12 +41,26 @@ protected DockerProviderService() { super(DockerProviderService.class.getSimpleName()); } - public void processArtifact(ContainerLauncher launcher, Component - component, SliderFileSystem fileSystem) throws IOException { + public void processArtifact(ContainerLauncher launcher, Application + application, RoleInstance roleInstance, SliderFileSystem fileSystem) + throws IOException { + Component component = roleInstance.providerRole.component; launcher.setYarnDockerMode(true); launcher.setDockerImage(component.getArtifact().getId()); launcher.setDockerNetwork(component.getConfiguration() .getProperty(DOCKER_NETWORK, DEFAULT_DOCKER_NETWORK)); + String domain = getConfig().get(RegistryConstants.KEY_DNS_DOMAIN); + String hostname; + if (domain == null || domain.isEmpty()) { + hostname = MessageFormat.format("{0}.{1}.{2}", roleInstance + .getCompInstanceName(), application.getName(), RegistryUtils + .currentUser()); + } else { + hostname = MessageFormat.format("{0}.{1}.{2}.{3}", roleInstance + .getCompInstanceName(), application.getName(), RegistryUtils + .currentUser(), domain); + } + launcher.setDockerHostname(hostname); launcher.setRunPrivilegedContainer(component.getRunPrivilegedContainer()); } } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/tarball/TarballProviderService.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/tarball/TarballProviderService.java index 9dd3499..8be63fd 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/tarball/TarballProviderService.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/tarball/TarballProviderService.java @@ -20,10 +20,12 @@ import org.apache.hadoop.fs.Path; import org.apache.hadoop.yarn.api.records.LocalResource; import org.apache.hadoop.yarn.api.records.LocalResourceType; +import org.apache.slider.api.resource.Application; import org.apache.slider.api.resource.Component; import org.apache.slider.common.tools.SliderFileSystem; import org.apache.slider.core.launch.ContainerLauncher; import org.apache.slider.providers.AbstractProviderService; +import org.apache.slider.server.appmaster.state.RoleInstance; import java.io.IOException; @@ -34,9 +36,11 @@ protected TarballProviderService() { } @Override - public void processArtifact(ContainerLauncher launcher, Component - component, SliderFileSystem fileSystem) throws IOException { - Path artifact = new Path(component.getArtifact().getId()); + public void processArtifact(ContainerLauncher launcher, Application + application, RoleInstance roleInstance, SliderFileSystem fileSystem) + throws IOException { + Path artifact = new Path(roleInstance.providerRole.component + .getArtifact().getId()); if (!fileSystem.isFile(artifact)) { throw new IOException("Package doesn't exist as a resource: " + artifact.toString()); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/state/RoleInstance.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/state/RoleInstance.java index 9ac26b5..5619492 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/state/RoleInstance.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/state/RoleInstance.java @@ -125,6 +125,7 @@ public RoleInstance(Container container, ProviderRole role) { } else { compInstanceName = role.name; } + compInstanceName = compInstanceName.toLowerCase().replaceAll("_", "-"); this.providerRole = role; } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-registry/src/main/java/org/apache/hadoop/registry/client/binding/RegistryPathUtils.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-registry/src/main/java/org/apache/hadoop/registry/client/binding/RegistryPathUtils.java index 5d8ea3f..5fa45f9 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-registry/src/main/java/org/apache/hadoop/registry/client/binding/RegistryPathUtils.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-registry/src/main/java/org/apache/hadoop/registry/client/binding/RegistryPathUtils.java @@ -213,6 +213,6 @@ public static String encodeForRegistry(String element) { * @return a string suitable for use in registry paths. */ public static String encodeYarnID(String yarnId) { - return yarnId.replace("_", "-"); + return yarnId.replace("container", "ctr").replace("_", "-"); } } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-registry/src/main/java/org/apache/hadoop/registry/client/types/Endpoint.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-registry/src/main/java/org/apache/hadoop/registry/client/types/Endpoint.java index 395f836..392884f 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-registry/src/main/java/org/apache/hadoop/registry/client/types/Endpoint.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-registry/src/main/java/org/apache/hadoop/registry/client/types/Endpoint.java @@ -19,7 +19,7 @@ package org.apache.hadoop.registry.client.types; import com.fasterxml.jackson.annotation.JsonIgnoreProperties; -import com.fasterxml.jackson.databind.annotation.JsonSerialize; +import com.fasterxml.jackson.annotation.JsonInclude; import com.google.common.base.Preconditions; import org.apache.hadoop.classification.InterfaceAudience; import org.apache.hadoop.classification.InterfaceStability; @@ -46,7 +46,7 @@ @InterfaceAudience.Public @InterfaceStability.Evolving @JsonIgnoreProperties(ignoreUnknown = true) -@JsonSerialize(include = JsonSerialize.Inclusion.NON_NULL) +@JsonInclude(JsonInclude.Include.NON_NULL) public final class Endpoint implements Cloneable { /** diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-registry/src/main/java/org/apache/hadoop/registry/client/types/ServiceRecord.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-registry/src/main/java/org/apache/hadoop/registry/client/types/ServiceRecord.java index 674d6d3..d40866a 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-registry/src/main/java/org/apache/hadoop/registry/client/types/ServiceRecord.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-registry/src/main/java/org/apache/hadoop/registry/client/types/ServiceRecord.java @@ -20,7 +20,7 @@ import com.fasterxml.jackson.annotation.JsonAnyGetter; import com.fasterxml.jackson.annotation.JsonAnySetter; -import com.fasterxml.jackson.databind.annotation.JsonSerialize; +import com.fasterxml.jackson.annotation.JsonInclude; import com.google.common.base.Preconditions; import org.apache.hadoop.classification.InterfaceAudience; import org.apache.hadoop.classification.InterfaceStability; @@ -37,7 +37,7 @@ */ @InterfaceAudience.Public @InterfaceStability.Evolving -@JsonSerialize(include = JsonSerialize.Inclusion.NON_NULL) +@JsonInclude(JsonInclude.Include.NON_NULL) public class ServiceRecord implements Cloneable { /** diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-registry/src/main/java/org/apache/hadoop/registry/server/dns/BaseServiceRecordProcessor.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-registry/src/main/java/org/apache/hadoop/registry/server/dns/BaseServiceRecordProcessor.java index 1289fb3..2fe3a6c 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-registry/src/main/java/org/apache/hadoop/registry/server/dns/BaseServiceRecordProcessor.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-registry/src/main/java/org/apache/hadoop/registry/server/dns/BaseServiceRecordProcessor.java @@ -285,7 +285,6 @@ public ContainerRecordDescriptor(String path, ServiceRecord record) */ protected Name getContainerIDName() throws TextParseException { String containerID = RegistryPathUtils.lastPathEntry(getPath()); - containerID = containerID.replace("container", "ctr"); return Name.fromString(String.format("%s.%s", containerID, domain)); } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-registry/src/main/java/org/apache/hadoop/registry/server/dns/ContainerServiceRecordProcessor.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-registry/src/main/java/org/apache/hadoop/registry/server/dns/ContainerServiceRecordProcessor.java index 75873d7..2e95f54 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-registry/src/main/java/org/apache/hadoop/registry/server/dns/ContainerServiceRecordProcessor.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-registry/src/main/java/org/apache/hadoop/registry/server/dns/ContainerServiceRecordProcessor.java @@ -16,6 +16,7 @@ */ package org.apache.hadoop.registry.server.dns; +import org.apache.hadoop.fs.PathNotFoundException; import org.apache.hadoop.registry.client.types.ServiceRecord; import org.apache.hadoop.registry.client.types.yarn.YarnRegistryAttributes; import org.xbill.DNS.Name; @@ -156,9 +157,11 @@ public TXTContainerRecordDescriptor(String path, */ @Override protected void init(ServiceRecord serviceRecord) { try { - this.setNames(new Name[] {getContainerIDName()}); + this.setNames(new Name[] {getContainerName()}); } catch (TextParseException e) { // log + } catch (PathNotFoundException e) { + // log } List txts = new ArrayList<>(); txts.add("id=" + serviceRecord.get(YarnRegistryAttributes.YARN_ID)); @@ -200,9 +203,11 @@ public PTRContainerRecordDescriptor(String path, } this.setNames(new Name[] {reverseLookupName}); try { - this.setTarget(getContainerIDName()); + this.setTarget(getContainerName()); } catch (TextParseException e) { //LOG + } catch (PathNotFoundException e) { + //LOG } } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-registry/src/test/java/org/apache/hadoop/registry/server/dns/TestRegistryDNS.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-registry/src/test/java/org/apache/hadoop/registry/server/dns/TestRegistryDNS.java index fcb602c..cc839cc 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-registry/src/test/java/org/apache/hadoop/registry/server/dns/TestRegistryDNS.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-registry/src/test/java/org/apache/hadoop/registry/server/dns/TestRegistryDNS.java @@ -111,7 +111,7 @@ + "}\n"; static final String CONTAINER_RECORD = "{\n" + " \"type\" : \"JSONServiceRecord\",\n" - + " \"description\" : \"YCLOUD\",\n" + + " \"description\" : \"COMP-NAME\",\n" + " \"external\" : [ ],\n" + " \"internal\" : [ ],\n" + " \"yarn:id\" : \"container_e50_1451931954322_0016_01_000002\",\n" @@ -122,7 +122,7 @@ private static final String CONTAINER_RECORD_NO_IP = "{\n" + " \"type\" : \"JSONServiceRecord\",\n" - + " \"description\" : \"YCLOUD\",\n" + + " \"description\" : \"COMP-NAME\",\n" + " \"external\" : [ ],\n" + " \"internal\" : [ ],\n" + " \"yarn:id\" : \"container_e50_1451931954322_0016_01_000002\",\n" @@ -131,7 +131,7 @@ private static final String CONTAINER_RECORD_YARN_PERSISTANCE_ABSENT = "{\n" + " \"type\" : \"JSONServiceRecord\",\n" - + " \"description\" : \"YCLOUD\",\n" + + " \"description\" : \"COMP-NAME\",\n" + " \"external\" : [ ],\n" + " \"internal\" : [ ],\n" + " \"yarn:id\" : \"container_e50_1451931954322_0016_01_000003\",\n" @@ -216,7 +216,7 @@ public void testContainerRegistration() throws Exception { CONTAINER_RECORD.getBytes()); getRegistryDNS().register( "/registry/users/root/services/org-apache-slider/test1/components/" - + "container-e50-1451931954322-0016-01-000002", + + "ctr-e50-1451931954322-0016-01-000002", record); // start assessing whether correct records are available @@ -225,7 +225,7 @@ public void testContainerRegistration() throws Exception { assertEquals("wrong result", "172.17.0.19", ((ARecord) recs[0]).getAddress().getHostAddress()); - recs = assertDNSQuery("ycloud.test1.root.hwx.test.", 1); + recs = assertDNSQuery("comp-name.test1.root.hwx.test.", 1); assertTrue("not an ARecord", recs[0] instanceof ARecord); } @@ -235,7 +235,7 @@ public void testContainerRegistrationPersistanceAbsent() throws Exception { CONTAINER_RECORD_YARN_PERSISTANCE_ABSENT.getBytes()); registryDNS.register( "/registry/users/root/services/org-apache-slider/test1/components/" - + "container-e50-1451931954322-0016-01-000003", + + "ctr-e50-1451931954322-0016-01-000003", record); Name name = @@ -254,7 +254,7 @@ public void testRecordTTL() throws Exception { CONTAINER_RECORD.getBytes()); getRegistryDNS().register( "/registry/users/root/services/org-apache-slider/test1/components/" - + "container-e50-1451931954322-0016-01-000002", + + "ctr-e50-1451931954322-0016-01-000002", record); // start assessing whether correct records are available @@ -264,7 +264,7 @@ public void testRecordTTL() throws Exception { ((ARecord) recs[0]).getAddress().getHostAddress()); assertEquals("wrong ttl", 30L, recs[0].getTTL()); - recs = assertDNSQuery("ycloud.test1.root.hwx.test.", 1); + recs = assertDNSQuery("comp-name.test1.root.hwx.test.", 1); assertTrue("not an ARecord", recs[0] instanceof ARecord); assertEquals("wrong ttl", 30L, recs[0].getTTL()); @@ -276,13 +276,13 @@ public void testReverseLookup() throws Exception { CONTAINER_RECORD.getBytes()); getRegistryDNS().register( "/registry/users/root/services/org-apache-slider/test1/components/" - + "container-e50-1451931954322-0016-01-000002", + + "ctr-e50-1451931954322-0016-01-000002", record); // start assessing whether correct records are available Record[] recs = assertDNSQuery("19.0.17.172.in-addr.arpa.", Type.PTR, 1); assertEquals("wrong result", - "ctr-e50-1451931954322-0016-01-000002.hwx.test.", + "comp-name.test1.root.hwx.test.", ((PTRRecord) recs[0]).getTarget().toString()); } @@ -302,13 +302,13 @@ public void testReverseLookupInLargeNetwork() throws Exception { CONTAINER_RECORD.getBytes()); getRegistryDNS().register( "/registry/users/root/services/org-apache-slider/test1/components/" - + "container-e50-1451931954322-0016-01-000002", + + "ctr-e50-1451931954322-0016-01-000002", record); // start assessing whether correct records are available Record[] recs = assertDNSQuery("19.0.17.172.in-addr.arpa.", Type.PTR, 1); assertEquals("wrong result", - "ctr-e50-1451931954322-0016-01-000002.hwx.test.", + "comp-name.test1.root.hwx.test.", ((PTRRecord) recs[0]).getTarget().toString()); } @@ -318,7 +318,7 @@ public void testMissingReverseLookup() throws Exception { CONTAINER_RECORD.getBytes()); getRegistryDNS().register( "/registry/users/root/services/org-apache-slider/test1/components/" - + "container-e50-1451931954322-0016-01-000002", + + "ctr-e50-1451931954322-0016-01-000002", record); // start assessing whether correct records are available @@ -339,7 +339,7 @@ public void testNoContainerIP() throws Exception { CONTAINER_RECORD_NO_IP.getBytes()); getRegistryDNS().register( "/registry/users/root/services/org-apache-slider/test1/components/" - + "container-e50-1451931954322-0016-01-000002", + + "ctr-e50-1451931954322-0016-01-000002", record); // start assessing whether correct records are available @@ -453,7 +453,7 @@ public void testAAAALookup() throws Exception { CONTAINER_RECORD.getBytes()); getRegistryDNS().register( "/registry/users/root/services/org-apache-slider/test1/components/" - + "container-e50-1451931954322-0016-01-000002", + + "ctr-e50-1451931954322-0016-01-000002", record); // start assessing whether correct records are available @@ -462,7 +462,7 @@ public void testAAAALookup() throws Exception { assertEquals("wrong result", "172.17.0.19", ((AAAARecord) recs[0]).getAddress().getHostAddress()); - recs = assertDNSQuery("ycloud.test1.root.hwx.test.", Type.AAAA, 1); + recs = assertDNSQuery("comp-name.test1.root.hwx.test.", Type.AAAA, 1); assertTrue("not an ARecord", recs[0] instanceof AAAARecord); } @@ -472,7 +472,7 @@ public void testNegativeLookup() throws Exception { CONTAINER_RECORD.getBytes()); getRegistryDNS().register( "/registry/users/root/services/org-apache-slider/test1/components/" - + "container-e50-1451931954322-0016-01-000002", + + "ctr-e50-1451931954322-0016-01-000002", record); // start assessing whether correct records are available @@ -528,7 +528,7 @@ public void testReadMasterFile() throws Exception { CONTAINER_RECORD.getBytes()); getRegistryDNS().register( "/registry/users/root/services/org-apache-slider/test1/components/" - + "container-e50-1451931954322-0016-01-000002", + + "ctr-e50-1451931954322-0016-01-000002", record); // start assessing whether correct records are available @@ -537,13 +537,13 @@ public void testReadMasterFile() throws Exception { assertEquals("wrong result", "172.17.0.19", ((ARecord) recs[0]).getAddress().getHostAddress()); - recs = assertDNSQuery("ycloud.test1.root.hwx.test.", 1); + recs = assertDNSQuery("comp-name.test1.root.hwx.test.", 1); assertTrue("not an ARecord", recs[0] instanceof ARecord); // lookup dyanmic reverse records recs = assertDNSQuery("19.0.17.172.in-addr.arpa.", Type.PTR, 1); assertEquals("wrong result", - "ctr-e50-1451931954322-0016-01-000002.hwx.test.", + "comp-name.test1.root.hwx.test.", ((PTRRecord) recs[0]).getTarget().toString()); // now lookup static reverse records diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/pom.xml b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/pom.xml index 75ec0a2..d8399ad 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/pom.xml +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/pom.xml @@ -52,6 +52,10 @@ hadoop-yarn-api + org.apache.hadoop + hadoop-yarn-registry + + javax.xml.bind jaxb-api diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/DockerLinuxContainerRuntime.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/DockerLinuxContainerRuntime.java index eba10a6..48c264a 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/DockerLinuxContainerRuntime.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/DockerLinuxContainerRuntime.java @@ -27,6 +27,8 @@ import org.apache.hadoop.classification.InterfaceStability; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.Path; +import org.apache.hadoop.registry.client.api.RegistryConstants; +import org.apache.hadoop.registry.client.binding.RegistryPathUtils; import org.apache.hadoop.security.UserGroupInformation; import org.apache.hadoop.security.authorize.AccessControlList; import org.apache.hadoop.util.StringUtils; @@ -101,6 +103,11 @@ * property. * *
  • + * {@code YARN_CONTAINER_RUNTIME_DOCKER_CONTAINER_HOSTNAME} sets the + * hostname to be used by the Docker container. If not specified, a + * hostname will be derived from the container ID. + *
  • + *
  • * {@code YARN_CONTAINER_RUNTIME_DOCKER_RUN_PRIVILEGED_CONTAINER} * controls whether the Docker container is a privileged container. In order * to use privileged containers, the @@ -134,6 +141,10 @@ "^(([a-zA-Z0-9.-]+)(:\\d+)?/)?([a-z0-9_./-]+)(:[\\w.-]+)?$"; private static final Pattern dockerImagePattern = Pattern.compile(DOCKER_IMAGE_PATTERN); + public static final String HOSTNAME_PATTERN = + "^[a-zA-Z0-9][a-zA-Z0-9_.-]+$"; + private static final Pattern hostnamePattern = Pattern.compile( + HOSTNAME_PATTERN); @InterfaceAudience.Private public static final String ENV_DOCKER_CONTAINER_IMAGE = @@ -147,6 +158,10 @@ @InterfaceAudience.Private public static final String ENV_DOCKER_CONTAINER_NETWORK = "YARN_CONTAINER_RUNTIME_DOCKER_CONTAINER_NETWORK"; + @InterfaceAudience.Private + public static final String ENV_DOCKER_CONTAINER_HOSTNAME = + "YARN_CONTAINER_RUNTIME_DOCKER_CONTAINER_HOSTNAME"; + @InterfaceAudience.Private public static final String ENV_DOCKER_CONTAINER_RUN_PRIVILEGED_CONTAINER = "YARN_CONTAINER_RUNTIME_DOCKER_RUN_PRIVILEGED_CONTAINER"; @InterfaceAudience.Private @@ -211,9 +226,7 @@ public DockerLinuxContainerRuntime(PrivilegedOperationExecutor this.privilegedOperationExecutor = privilegedOperationExecutor; if (cGroupsHandler == null) { - if (LOG.isInfoEnabled()) { - LOG.info("cGroupsHandler is null - cgroups not in use."); - } + LOG.info("cGroupsHandler is null - cgroups not in use."); } else { this.cGroupsHandler = cGroupsHandler; } @@ -267,6 +280,35 @@ private void validateContainerNetworkType(String network) throw new ContainerExecutionException(msg); } + public static void validateHostname(String hostname) throws + ContainerExecutionException { + if (hostname != null && !hostname.isEmpty()) { + if (!hostnamePattern.matcher(hostname).matches()) { + throw new ContainerExecutionException("Hostname '" + hostname + + "' doesn't match docker hostname pattern"); + } + } + } + + /** Set a DNS friendly hostname. */ + private void setHostname(DockerRunCommand runCommand, String + containerIdStr, String name) + throws ContainerExecutionException { + if (name == null || name.isEmpty()) { + name = RegistryPathUtils.encodeYarnID(containerIdStr); + + String domain = conf.get(RegistryConstants.KEY_DNS_DOMAIN); + + if (domain != null) { + name += ("." + domain); + } + validateHostname(name); + } + + LOG.info("setting hostname in container to: " + name); + runCommand.setHostname(name); + } + /** * If CGROUPS in enabled and not set to none, then set the CGROUP parent for * the command instance. @@ -343,10 +385,8 @@ private boolean allowPrivilegedContainerExecution(Container container) return false; } - if (LOG.isInfoEnabled()) { - LOG.info("Privileged container requested for : " + container - .getContainerId().toString()); - } + LOG.info("Privileged container requested for : " + container + .getContainerId().toString()); //Ok, so we have been asked to run a privileged container. Security // checks need to be run. Each violation is an error. @@ -375,10 +415,8 @@ private boolean allowPrivilegedContainerExecution(Container container) throw new ContainerExecutionException(message); } - if (LOG.isInfoEnabled()) { - LOG.info("All checks pass. Launching privileged container for : " - + container.getContainerId().toString()); - } + LOG.info("All checks pass. Launching privileged container for : " + + container.getContainerId().toString()); return true; } @@ -413,6 +451,7 @@ public void launchContainer(ContainerRuntimeContext ctx) .getEnvironment(); String imageName = environment.get(ENV_DOCKER_CONTAINER_IMAGE); String network = environment.get(ENV_DOCKER_CONTAINER_NETWORK); + String hostname = environment.get(ENV_DOCKER_CONTAINER_HOSTNAME); if(network == null || network.isEmpty()) { network = defaultNetwork; @@ -420,6 +459,8 @@ public void launchContainer(ContainerRuntimeContext ctx) validateContainerNetworkType(network); + validateHostname(hostname); + validateImageName(imageName); String containerIdStr = container.getContainerId().toString(); @@ -450,12 +491,13 @@ public void launchContainer(ContainerRuntimeContext ctx) runAsUser, imageName) .detachOnRun() .setContainerWorkDir(containerWorkDir.toString()) - .setNetworkType(network) - .setCapabilities(capabilities) + .setNetworkType(network); + setHostname(runCommand, containerIdStr, hostname); + runCommand.setCapabilities(capabilities) .addMountLocation(CGROUPS_ROOT_DIRECTORY, CGROUPS_ROOT_DIRECTORY + ":ro", false); - List allDirs = new ArrayList<>(containerLocalDirs); + List allDirs = new ArrayList<>(containerLocalDirs); allDirs.addAll(filecacheDirs); allDirs.add(containerWorkDir.toString()); allDirs.addAll(containerLogDirs); @@ -493,9 +535,7 @@ public void launchContainer(ContainerRuntimeContext ctx) ENV_DOCKER_CONTAINER_RUN_OVERRIDE_DISABLE); if (disableOverride != null && disableOverride.equals("true")) { - if (LOG.isInfoEnabled()) { - LOG.info("command override disabled"); - } + LOG.info("command override disabled"); } else { List overrideCommands = new ArrayList<>(); Path launchDst = diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/docker/DockerRunCommand.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/docker/DockerRunCommand.java index f79f4ed..b645754 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/docker/DockerRunCommand.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/docker/DockerRunCommand.java @@ -91,6 +91,12 @@ public DockerRunCommand setCapabilities(Set capabilties) { return this; } + + public DockerRunCommand setHostname(String hostname) { + super.addCommandArguments("--hostname=" + hostname); + return this; + } + public DockerRunCommand addDevice(String sourceDevice, String destinationDevice) { super.addCommandArguments("--device=" + sourceDevice + ":" + diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/container-executor.c b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/container-executor.c index 5d138f3..5070d62 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/container-executor.c +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/container-executor.c @@ -1215,6 +1215,7 @@ char* sanitize_docker_command(const char *line) { {"rm", no_argument, 0, 'r' }, {"workdir", required_argument, 0, 'w' }, {"net", required_argument, 0, 'e' }, + {"hostname", required_argument, 0, 'h' }, {"cgroup-parent", required_argument, 0, 'g' }, {"privileged", no_argument, 0, 'p' }, {"cap-add", required_argument, 0, 'a' }, @@ -1256,6 +1257,9 @@ char* sanitize_docker_command(const char *line) { case 'e': quote_and_append_arg(&output, &output_size, "--net=", optarg); break; + case 'h': + quote_and_append_arg(&output, &output_size, "--hostname=", optarg); + break; case 'v': quote_and_append_arg(&output, &output_size, "-v ", optarg); break; diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/test/test-container-executor.c b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/test/test-container-executor.c index 83d11ec..b7d0e44 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/test/test-container-executor.c +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/test/test-container-executor.c @@ -1088,17 +1088,17 @@ void test_trim_function() { void test_sanitize_docker_command() { char *input[] = { - "run --name=cname --user=nobody -d --workdir=/yarn/local/cdir --privileged --rm --device=/sys/fs/cgroup/device:/sys/fs/cgroup/device --detach=true --cgroup-parent=/sys/fs/cgroup/cpu/yarn/cid --net=host --cap-drop=ALL --cap-add=SYS_CHROOT --cap-add=MKNOD --cap-add=SETFCAP --cap-add=SETPCAP --cap-add=FSETID --cap-add=CHOWN --cap-add=AUDIT_WRITE --cap-add=SETGID --cap-add=NET_RAW --cap-add=FOWNER --cap-add=SETUID --cap-add=DAC_OVERRIDE --cap-add=KILL --cap-add=NET_BIND_SERVICE -v /sys/fs/cgroup:/sys/fs/cgroup:ro -v /yarn/local/cdir:/yarn/local/cdir -v /yarn/local/usercache/test/:/yarn/local/usercache/test/ ubuntu bash /yarn/local/usercache/test/appcache/aid/cid/launch_container.sh", - "run --name=$CID --user=nobody -d --workdir=/yarn/local/cdir --privileged --rm --device=/sys/fs/cgroup/device:/sys/fs/cgroup/device --detach=true --cgroup-parent=/sys/fs/cgroup/cpu/yarn/cid --net=host --cap-drop=ALL --cap-add=SYS_CHROOT --cap-add=MKNOD --cap-add=SETFCAP --cap-add=SETPCAP --cap-add=FSETID --cap-add=CHOWN --cap-add=AUDIT_WRITE --cap-add=SETGID --cap-add=NET_RAW --cap-add=FOWNER --cap-add=SETUID --cap-add=DAC_OVERRIDE --cap-add=KILL --cap-add=NET_BIND_SERVICE -v /sys/fs/cgroup:/sys/fs/cgroup:ro -v /yarn/local/cdir:/yarn/local/cdir -v /yarn/local/usercache/test/:/yarn/local/usercache/test/ ubuntu bash /yarn/local/usercache/test/appcache/aid/cid/launch_container.sh", - "run --name=cname --user=nobody -d --workdir=/yarn/local/cdir --privileged --rm --device=/sys/fs/cgroup/device:/sys/fs/cgroup/device --detach=true --cgroup-parent=/sys/fs/cgroup/cpu/yarn/cid --net=host --cap-drop=ALL --cap-add=SYS_CHROOT --cap-add=MKNOD --cap-add=SETFCAP --cap-add=SETPCAP --cap-add=FSETID --cap-add=CHOWN --cap-add=AUDIT_WRITE --cap-add=SETGID --cap-add=NET_RAW --cap-add=FOWNER --cap-add=SETUID --cap-add=DAC_OVERRIDE --cap-add=KILL --cap-add=NET_BIND_SERVICE -v /sys/fs/cgroup:/sys/fs/cgroup:ro -v /yarn/local/cdir:/yarn/local/cdir -v /yarn/local/usercache/test/:/yarn/local/usercache/test/ ubuntu || touch /tmp/file # bash /yarn/local/usercache/test/appcache/aid/cid/launch_container.sh", - "run --name=cname --user=nobody -d --workdir=/yarn/local/cdir --privileged --rm --device=/sys/fs/cgroup/device:/sys/fs/cgroup/device --detach=true --cgroup-parent=/sys/fs/cgroup/cpu/yarn/cid --net=host --cap-drop=ALL --cap-add=SYS_CHROOT --cap-add=MKNOD --cap-add=SETFCAP --cap-add=SETPCAP --cap-add=FSETID --cap-add=CHOWN --cap-add=AUDIT_WRITE --cap-add=SETGID --cap-add=NET_RAW --cap-add=FOWNER --cap-add=SETUID --cap-add=DAC_OVERRIDE --cap-add=KILL --cap-add=NET_BIND_SERVICE -v /sys/fs/cgroup:/sys/fs/cgroup:ro -v /yarn/local/cdir:/yarn/local/cdir -v /yarn/local/usercache/test/:/yarn/local/usercache/test/ ubuntu' || touch /tmp/file # bash /yarn/local/usercache/test/appcache/aid/cid/launch_container.sh", + "run --name=cname --user=nobody -d --workdir=/yarn/local/cdir --privileged --rm --device=/sys/fs/cgroup/device:/sys/fs/cgroup/device --detach=true --cgroup-parent=/sys/fs/cgroup/cpu/yarn/cid --net=host --hostname=test.host.name --cap-drop=ALL --cap-add=SYS_CHROOT --cap-add=MKNOD --cap-add=SETFCAP --cap-add=SETPCAP --cap-add=FSETID --cap-add=CHOWN --cap-add=AUDIT_WRITE --cap-add=SETGID --cap-add=NET_RAW --cap-add=FOWNER --cap-add=SETUID --cap-add=DAC_OVERRIDE --cap-add=KILL --cap-add=NET_BIND_SERVICE -v /sys/fs/cgroup:/sys/fs/cgroup:ro -v /yarn/local/cdir:/yarn/local/cdir -v /yarn/local/usercache/test/:/yarn/local/usercache/test/ ubuntu bash /yarn/local/usercache/test/appcache/aid/cid/launch_container.sh", + "run --name=$CID --user=nobody -d --workdir=/yarn/local/cdir --privileged --rm --device=/sys/fs/cgroup/device:/sys/fs/cgroup/device --detach=true --cgroup-parent=/sys/fs/cgroup/cpu/yarn/cid --net=host --hostname=test.host.name --cap-drop=ALL --cap-add=SYS_CHROOT --cap-add=MKNOD --cap-add=SETFCAP --cap-add=SETPCAP --cap-add=FSETID --cap-add=CHOWN --cap-add=AUDIT_WRITE --cap-add=SETGID --cap-add=NET_RAW --cap-add=FOWNER --cap-add=SETUID --cap-add=DAC_OVERRIDE --cap-add=KILL --cap-add=NET_BIND_SERVICE -v /sys/fs/cgroup:/sys/fs/cgroup:ro -v /yarn/local/cdir:/yarn/local/cdir -v /yarn/local/usercache/test/:/yarn/local/usercache/test/ ubuntu bash /yarn/local/usercache/test/appcache/aid/cid/launch_container.sh", + "run --name=cname --user=nobody -d --workdir=/yarn/local/cdir --privileged --rm --device=/sys/fs/cgroup/device:/sys/fs/cgroup/device --detach=true --cgroup-parent=/sys/fs/cgroup/cpu/yarn/cid --net=host --hostname=test.host.name --cap-drop=ALL --cap-add=SYS_CHROOT --cap-add=MKNOD --cap-add=SETFCAP --cap-add=SETPCAP --cap-add=FSETID --cap-add=CHOWN --cap-add=AUDIT_WRITE --cap-add=SETGID --cap-add=NET_RAW --cap-add=FOWNER --cap-add=SETUID --cap-add=DAC_OVERRIDE --cap-add=KILL --cap-add=NET_BIND_SERVICE -v /sys/fs/cgroup:/sys/fs/cgroup:ro -v /yarn/local/cdir:/yarn/local/cdir -v /yarn/local/usercache/test/:/yarn/local/usercache/test/ ubuntu || touch /tmp/file # bash /yarn/local/usercache/test/appcache/aid/cid/launch_container.sh", + "run --name=cname --user=nobody -d --workdir=/yarn/local/cdir --privileged --rm --device=/sys/fs/cgroup/device:/sys/fs/cgroup/device --detach=true --cgroup-parent=/sys/fs/cgroup/cpu/yarn/cid --net=host --hostname=test.host.name --cap-drop=ALL --cap-add=SYS_CHROOT --cap-add=MKNOD --cap-add=SETFCAP --cap-add=SETPCAP --cap-add=FSETID --cap-add=CHOWN --cap-add=AUDIT_WRITE --cap-add=SETGID --cap-add=NET_RAW --cap-add=FOWNER --cap-add=SETUID --cap-add=DAC_OVERRIDE --cap-add=KILL --cap-add=NET_BIND_SERVICE -v /sys/fs/cgroup:/sys/fs/cgroup:ro -v /yarn/local/cdir:/yarn/local/cdir -v /yarn/local/usercache/test/:/yarn/local/usercache/test/ ubuntu' || touch /tmp/file # bash /yarn/local/usercache/test/appcache/aid/cid/launch_container.sh", "run ''''''''" }; char *expected_output[] = { - "run --name='cname' --user='nobody' -d --workdir='/yarn/local/cdir' --privileged --rm --device='/sys/fs/cgroup/device:/sys/fs/cgroup/device' --detach='true' --cgroup-parent='/sys/fs/cgroup/cpu/yarn/cid' --net='host' --cap-drop='ALL' --cap-add='SYS_CHROOT' --cap-add='MKNOD' --cap-add='SETFCAP' --cap-add='SETPCAP' --cap-add='FSETID' --cap-add='CHOWN' --cap-add='AUDIT_WRITE' --cap-add='SETGID' --cap-add='NET_RAW' --cap-add='FOWNER' --cap-add='SETUID' --cap-add='DAC_OVERRIDE' --cap-add='KILL' --cap-add='NET_BIND_SERVICE' -v '/sys/fs/cgroup:/sys/fs/cgroup:ro' -v '/yarn/local/cdir:/yarn/local/cdir' -v '/yarn/local/usercache/test/:/yarn/local/usercache/test/' 'ubuntu' 'bash' '/yarn/local/usercache/test/appcache/aid/cid/launch_container.sh' ", - "run --name='$CID' --user='nobody' -d --workdir='/yarn/local/cdir' --privileged --rm --device='/sys/fs/cgroup/device:/sys/fs/cgroup/device' --detach='true' --cgroup-parent='/sys/fs/cgroup/cpu/yarn/cid' --net='host' --cap-drop='ALL' --cap-add='SYS_CHROOT' --cap-add='MKNOD' --cap-add='SETFCAP' --cap-add='SETPCAP' --cap-add='FSETID' --cap-add='CHOWN' --cap-add='AUDIT_WRITE' --cap-add='SETGID' --cap-add='NET_RAW' --cap-add='FOWNER' --cap-add='SETUID' --cap-add='DAC_OVERRIDE' --cap-add='KILL' --cap-add='NET_BIND_SERVICE' -v '/sys/fs/cgroup:/sys/fs/cgroup:ro' -v '/yarn/local/cdir:/yarn/local/cdir' -v '/yarn/local/usercache/test/:/yarn/local/usercache/test/' 'ubuntu' 'bash' '/yarn/local/usercache/test/appcache/aid/cid/launch_container.sh' ", - "run --name='cname' --user='nobody' -d --workdir='/yarn/local/cdir' --privileged --rm --device='/sys/fs/cgroup/device:/sys/fs/cgroup/device' --detach='true' --cgroup-parent='/sys/fs/cgroup/cpu/yarn/cid' --net='host' --cap-drop='ALL' --cap-add='SYS_CHROOT' --cap-add='MKNOD' --cap-add='SETFCAP' --cap-add='SETPCAP' --cap-add='FSETID' --cap-add='CHOWN' --cap-add='AUDIT_WRITE' --cap-add='SETGID' --cap-add='NET_RAW' --cap-add='FOWNER' --cap-add='SETUID' --cap-add='DAC_OVERRIDE' --cap-add='KILL' --cap-add='NET_BIND_SERVICE' -v '/sys/fs/cgroup:/sys/fs/cgroup:ro' -v '/yarn/local/cdir:/yarn/local/cdir' -v '/yarn/local/usercache/test/:/yarn/local/usercache/test/' 'ubuntu' '||' 'touch' '/tmp/file' '#' 'bash' '/yarn/local/usercache/test/appcache/aid/cid/launch_container.sh' ", - "run --name='cname' --user='nobody' -d --workdir='/yarn/local/cdir' --privileged --rm --device='/sys/fs/cgroup/device:/sys/fs/cgroup/device' --detach='true' --cgroup-parent='/sys/fs/cgroup/cpu/yarn/cid' --net='host' --cap-drop='ALL' --cap-add='SYS_CHROOT' --cap-add='MKNOD' --cap-add='SETFCAP' --cap-add='SETPCAP' --cap-add='FSETID' --cap-add='CHOWN' --cap-add='AUDIT_WRITE' --cap-add='SETGID' --cap-add='NET_RAW' --cap-add='FOWNER' --cap-add='SETUID' --cap-add='DAC_OVERRIDE' --cap-add='KILL' --cap-add='NET_BIND_SERVICE' -v '/sys/fs/cgroup:/sys/fs/cgroup:ro' -v '/yarn/local/cdir:/yarn/local/cdir' -v '/yarn/local/usercache/test/:/yarn/local/usercache/test/' 'ubuntu'\"'\"'' '||' 'touch' '/tmp/file' '#' 'bash' '/yarn/local/usercache/test/appcache/aid/cid/launch_container.sh' ", + "run --name='cname' --user='nobody' -d --workdir='/yarn/local/cdir' --privileged --rm --device='/sys/fs/cgroup/device:/sys/fs/cgroup/device' --detach='true' --cgroup-parent='/sys/fs/cgroup/cpu/yarn/cid' --net='host' --hostname='test.host.name' --cap-drop='ALL' --cap-add='SYS_CHROOT' --cap-add='MKNOD' --cap-add='SETFCAP' --cap-add='SETPCAP' --cap-add='FSETID' --cap-add='CHOWN' --cap-add='AUDIT_WRITE' --cap-add='SETGID' --cap-add='NET_RAW' --cap-add='FOWNER' --cap-add='SETUID' --cap-add='DAC_OVERRIDE' --cap-add='KILL' --cap-add='NET_BIND_SERVICE' -v '/sys/fs/cgroup:/sys/fs/cgroup:ro' -v '/yarn/local/cdir:/yarn/local/cdir' -v '/yarn/local/usercache/test/:/yarn/local/usercache/test/' 'ubuntu' 'bash' '/yarn/local/usercache/test/appcache/aid/cid/launch_container.sh' ", + "run --name='$CID' --user='nobody' -d --workdir='/yarn/local/cdir' --privileged --rm --device='/sys/fs/cgroup/device:/sys/fs/cgroup/device' --detach='true' --cgroup-parent='/sys/fs/cgroup/cpu/yarn/cid' --net='host' --hostname='test.host.name' --cap-drop='ALL' --cap-add='SYS_CHROOT' --cap-add='MKNOD' --cap-add='SETFCAP' --cap-add='SETPCAP' --cap-add='FSETID' --cap-add='CHOWN' --cap-add='AUDIT_WRITE' --cap-add='SETGID' --cap-add='NET_RAW' --cap-add='FOWNER' --cap-add='SETUID' --cap-add='DAC_OVERRIDE' --cap-add='KILL' --cap-add='NET_BIND_SERVICE' -v '/sys/fs/cgroup:/sys/fs/cgroup:ro' -v '/yarn/local/cdir:/yarn/local/cdir' -v '/yarn/local/usercache/test/:/yarn/local/usercache/test/' 'ubuntu' 'bash' '/yarn/local/usercache/test/appcache/aid/cid/launch_container.sh' ", + "run --name='cname' --user='nobody' -d --workdir='/yarn/local/cdir' --privileged --rm --device='/sys/fs/cgroup/device:/sys/fs/cgroup/device' --detach='true' --cgroup-parent='/sys/fs/cgroup/cpu/yarn/cid' --net='host' --hostname='test.host.name' --cap-drop='ALL' --cap-add='SYS_CHROOT' --cap-add='MKNOD' --cap-add='SETFCAP' --cap-add='SETPCAP' --cap-add='FSETID' --cap-add='CHOWN' --cap-add='AUDIT_WRITE' --cap-add='SETGID' --cap-add='NET_RAW' --cap-add='FOWNER' --cap-add='SETUID' --cap-add='DAC_OVERRIDE' --cap-add='KILL' --cap-add='NET_BIND_SERVICE' -v '/sys/fs/cgroup:/sys/fs/cgroup:ro' -v '/yarn/local/cdir:/yarn/local/cdir' -v '/yarn/local/usercache/test/:/yarn/local/usercache/test/' 'ubuntu' '||' 'touch' '/tmp/file' '#' 'bash' '/yarn/local/usercache/test/appcache/aid/cid/launch_container.sh' ", + "run --name='cname' --user='nobody' -d --workdir='/yarn/local/cdir' --privileged --rm --device='/sys/fs/cgroup/device:/sys/fs/cgroup/device' --detach='true' --cgroup-parent='/sys/fs/cgroup/cpu/yarn/cid' --net='host' --hostname='test.host.name' --cap-drop='ALL' --cap-add='SYS_CHROOT' --cap-add='MKNOD' --cap-add='SETFCAP' --cap-add='SETPCAP' --cap-add='FSETID' --cap-add='CHOWN' --cap-add='AUDIT_WRITE' --cap-add='SETGID' --cap-add='NET_RAW' --cap-add='FOWNER' --cap-add='SETUID' --cap-add='DAC_OVERRIDE' --cap-add='KILL' --cap-add='NET_BIND_SERVICE' -v '/sys/fs/cgroup:/sys/fs/cgroup:ro' -v '/yarn/local/cdir:/yarn/local/cdir' -v '/yarn/local/usercache/test/:/yarn/local/usercache/test/' 'ubuntu'\"'\"'' '||' 'touch' '/tmp/file' '#' 'bash' '/yarn/local/usercache/test/appcache/aid/cid/launch_container.sh' ", "run ''\"'\"''\"'\"''\"'\"''\"'\"''\"'\"''\"'\"''\"'\"''\"'\"'' ", }; diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/TestDockerContainerRuntime.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/TestDockerContainerRuntime.java index f611843..9894dcd 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/TestDockerContainerRuntime.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/TestDockerContainerRuntime.java @@ -25,6 +25,7 @@ import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.FileUtil; import org.apache.hadoop.fs.Path; +import org.apache.hadoop.registry.client.binding.RegistryPathUtils; import org.apache.hadoop.yarn.api.records.ContainerId; import org.apache.hadoop.yarn.api.records.ContainerLaunchContext; import org.apache.hadoop.yarn.conf.YarnConfiguration; @@ -69,6 +70,7 @@ private PrivilegedOperationExecutor mockExecutor; private CGroupsHandler mockCGroupsHandler; private String containerId; + private String defaultHostname; private Container container; private ContainerId cId; private ContainerLaunchContext context; @@ -108,6 +110,7 @@ public void setup() { .mock(PrivilegedOperationExecutor.class); mockCGroupsHandler = Mockito.mock(CGroupsHandler.class); containerId = "container_id"; + defaultHostname = RegistryPathUtils.encodeYarnID(containerId); container = mock(Container.class); cId = mock(ContainerId.class); context = mock(ContainerLaunchContext.class); @@ -287,6 +290,7 @@ public void testDockerContainerLaunch() .append("--user=%2$s -d ") .append("--workdir=%3$s ") .append("--net=host ") + .append("--hostname=" + defaultHostname + " ") .append(getExpectedTestCapabilitiesArgumentString()) .append(getExpectedCGroupsMountString()) .append("-v %4$s:%4$s ") @@ -365,7 +369,7 @@ public void testContainerLaunchWithNetworkingDefaults() String disallowedNetwork = "sdn" + Integer.toString(randEngine.nextInt()); try { - env.put("YARN_CONTAINER_RUNTIME_DOCKER_CONTAINER_NETWORK", + env.put(DockerLinuxContainerRuntime.ENV_DOCKER_CONTAINER_NETWORK, disallowedNetwork); runtime.launchContainer(builder.build()); Assert.fail("Network was expected to be disallowed: " + @@ -378,8 +382,11 @@ public void testContainerLaunchWithNetworkingDefaults() .DEFAULT_NM_DOCKER_ALLOWED_CONTAINER_NETWORKS.length; String allowedNetwork = YarnConfiguration .DEFAULT_NM_DOCKER_ALLOWED_CONTAINER_NETWORKS[randEngine.nextInt(size)]; - env.put("YARN_CONTAINER_RUNTIME_DOCKER_CONTAINER_NETWORK", + env.put(DockerLinuxContainerRuntime.ENV_DOCKER_CONTAINER_NETWORK, allowedNetwork); + String expectedHostname = "test.hostname"; + env.put(DockerLinuxContainerRuntime.ENV_DOCKER_CONTAINER_HOSTNAME, + expectedHostname); //this should cause no failures. @@ -393,6 +400,7 @@ public void testContainerLaunchWithNetworkingDefaults() new StringBuffer("run --name=%1$s ").append("--user=%2$s -d ") .append("--workdir=%3$s ") .append("--net=" + allowedNetwork + " ") + .append("--hostname=" + expectedHostname + " ") .append(getExpectedTestCapabilitiesArgumentString()) .append(getExpectedCGroupsMountString()) .append("-v %4$s:%4$s ").append("-v %5$s:%5$s ") @@ -448,6 +456,7 @@ public void testContainerLaunchWithCustomNetworks() new StringBuffer("run --name=%1$s ").append("--user=%2$s -d ") .append("--workdir=%3$s ") .append("--net=" + customNetwork1 + " ") + .append("--hostname=" + defaultHostname + " ") .append(getExpectedTestCapabilitiesArgumentString()) .append(getExpectedCGroupsMountString()) .append("-v %4$s:%4$s ").append("-v %5$s:%5$s ") @@ -471,7 +480,7 @@ public void testContainerLaunchWithCustomNetworks() //now set an explicit (non-default) allowedNetwork and ensure that it is // used. - env.put("YARN_CONTAINER_RUNTIME_DOCKER_CONTAINER_NETWORK", + env.put(DockerLinuxContainerRuntime.ENV_DOCKER_CONTAINER_NETWORK, customNetwork2); runtime.launchContainer(builder.build()); @@ -485,6 +494,7 @@ public void testContainerLaunchWithCustomNetworks() new StringBuffer("run --name=%1$s ").append("--user=%2$s -d ") .append("--workdir=%3$s ") .append("--net=" + customNetwork2 + " ") + .append("--hostname=" + defaultHostname + " ") .append(getExpectedTestCapabilitiesArgumentString()) .append(getExpectedCGroupsMountString()) .append("-v %4$s:%4$s ").append("-v %5$s:%5$s ") @@ -505,7 +515,7 @@ public void testContainerLaunchWithCustomNetworks() //disallowed network should trigger a launch failure - env.put("YARN_CONTAINER_RUNTIME_DOCKER_CONTAINER_NETWORK", + env.put(DockerLinuxContainerRuntime.ENV_DOCKER_CONTAINER_NETWORK, customNetwork3); try { runtime.launchContainer(builder.build()); @@ -524,8 +534,8 @@ public void testLaunchPrivilegedContainersInvalidEnvVar() mockExecutor, mockCGroupsHandler); runtime.initialize(conf); - env.put("YARN_CONTAINER_RUNTIME_DOCKER_RUN_PRIVILEGED_CONTAINER", - "invalid-value"); + env.put(DockerLinuxContainerRuntime + .ENV_DOCKER_CONTAINER_RUN_PRIVILEGED_CONTAINER, "invalid-value"); runtime.launchContainer(builder.build()); PrivilegedOperation op = capturePrivilegedOperationAndVerifyArgs(); @@ -552,8 +562,8 @@ public void testLaunchPrivilegedContainersWithDisabledSetting() mockExecutor, mockCGroupsHandler); runtime.initialize(conf); - env.put("YARN_CONTAINER_RUNTIME_DOCKER_RUN_PRIVILEGED_CONTAINER", - "true"); + env.put(DockerLinuxContainerRuntime + .ENV_DOCKER_CONTAINER_RUN_PRIVILEGED_CONTAINER, "true"); try { runtime.launchContainer(builder.build()); @@ -575,8 +585,8 @@ public void testLaunchPrivilegedContainersWithEnabledSettingAndDefaultACL() mockExecutor, mockCGroupsHandler); runtime.initialize(conf); - env.put("YARN_CONTAINER_RUNTIME_DOCKER_RUN_PRIVILEGED_CONTAINER", - "true"); + env.put(DockerLinuxContainerRuntime + .ENV_DOCKER_CONTAINER_RUN_PRIVILEGED_CONTAINER, "true"); //By default // yarn.nodemanager.runtime.linux.docker.privileged-containers.acl // is empty. So we expect this launch to fail. @@ -605,8 +615,8 @@ public void testLaunchPrivilegedContainersWithEnabledSettingAndDefaultACL() mockExecutor, mockCGroupsHandler); runtime.initialize(conf); - env.put("YARN_CONTAINER_RUNTIME_DOCKER_RUN_PRIVILEGED_CONTAINER", - "true"); + env.put(DockerLinuxContainerRuntime + .ENV_DOCKER_CONTAINER_RUN_PRIVILEGED_CONTAINER, "true"); try { runtime.launchContainer(builder.build()); @@ -632,8 +642,8 @@ public void testLaunchPrivilegedContainersWithEnabledSettingAndDefaultACL() mockExecutor, mockCGroupsHandler); runtime.initialize(conf); - env.put("YARN_CONTAINER_RUNTIME_DOCKER_RUN_PRIVILEGED_CONTAINER", - "true"); + env.put(DockerLinuxContainerRuntime + .ENV_DOCKER_CONTAINER_RUN_PRIVILEGED_CONTAINER, "true"); runtime.launchContainer(builder.build()); PrivilegedOperation op = capturePrivilegedOperationAndVerifyArgs(); @@ -927,4 +937,24 @@ public void testDockerImageNamePattern() throws Exception { } } } + + @Test + public void testDockerHostnamePattern() throws Exception { + String[] validNames = {"ab", "a.b.c.d", "a1-b.cd.ef", "0AB.", "C_D-"}; + + String[] invalidNames = {"a", "a#.b.c", "-a.b.c", "a@b.c", "a/b/c"}; + + for (String name : validNames) { + DockerLinuxContainerRuntime.validateHostname(name); + } + + for (String name : invalidNames) { + try { + DockerLinuxContainerRuntime.validateHostname(name); + Assert.fail(name + " is an invalid hostname and should fail the regex"); + } catch (ContainerExecutionException ce) { + continue; + } + } + } }