From 0eff639ca518d735c8d16f24eec852ba28800dea Mon Sep 17 00:00:00 2001 From: Prabhu Joseph Date: Fri, 4 Oct 2019 02:06:47 +0530 Subject: [PATCH] YARN-9860. Enable service mode for Docker containers on YARN. 1. This introduces service mode for Docker container which allows injecting configuration. 2. This adds new field visibility for files to be localized in YARN Native Service. Signed-off-by: Prabhu Joseph --- .../yarn/service/api/records/ConfigFile.java | 28 ++++- .../hadoop/yarn/service/client/ServiceClient.java | 20 +++- .../yarn/service/conf/YarnServiceConstants.java | 2 + .../yarn/service/provider/ProviderUtils.java | 41 ++++++- .../provider/tarball/TarballProviderService.java | 4 +- .../hadoop/yarn/service/utils/CoreFileSystem.java | 17 ++- .../yarn/service/utils/SliderFileSystem.java | 34 ++++++ .../yarn/service/provider/TestProviderUtils.java | 119 +++++++++++---------- .../linux/runtime/DockerLinuxContainerRuntime.java | 39 +++++-- .../linux/runtime/docker/DockerRunCommand.java | 6 ++ .../container-executor/impl/container-executor.h | 6 -- .../container-executor/impl/utils/docker-util.c | 59 +++++++--- .../container-executor/impl/utils/docker-util.h | 4 +- .../src/site/markdown/DockerContainers.md | 23 ++++ 14 files changed, 305 insertions(+), 97 deletions(-) diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/api/records/ConfigFile.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/api/records/ConfigFile.java index c09373f..060e204 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/api/records/ConfigFile.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/api/records/ConfigFile.java @@ -24,6 +24,7 @@ import io.swagger.annotations.ApiModelProperty; import org.apache.hadoop.classification.InterfaceAudience; import org.apache.hadoop.classification.InterfaceStability; +import org.apache.hadoop.yarn.api.records.LocalResourceVisibility; import javax.xml.bind.annotation.XmlElement; import javax.xml.bind.annotation.XmlEnum; @@ -73,6 +74,7 @@ public String toString() { private TypeEnum type = null; private String destFile = null; private String srcFile = null; + private LocalResourceVisibility visibility = null; private Map properties = new HashMap<>(); public ConfigFile copy() { @@ -80,6 +82,7 @@ public ConfigFile copy() { copy.setType(this.getType()); copy.setSrcFile(this.getSrcFile()); copy.setDestFile(this.getDestFile()); + copy.setVisibility(this.visibility); if (this.getProperties() != null && !this.getProperties().isEmpty()) { copy.getProperties().putAll(this.getProperties()); } @@ -150,6 +153,26 @@ public void setSrcFile(String srcFile) { this.srcFile = srcFile; } + + /** + * Visibility of the Config file. + **/ + public ConfigFile visibility(LocalResourceVisibility localrsrcVisibility) { + this.visibility = localrsrcVisibility; + return this; + } + + @ApiModelProperty(example = "null", value = "Visibility of the Config file") + @JsonProperty("visibility") + public LocalResourceVisibility getVisibility() { + return visibility; + } + + @XmlElement(name = "visibility", defaultValue="APPLICATION") + public void setVisibility(LocalResourceVisibility localrsrcVisibility) { + this.visibility = localrsrcVisibility; + } + /** A blob of key value pairs that will be dumped in the dest_file in the format as specified in type. If src_file is specified, src_file content are dumped @@ -200,12 +223,13 @@ public boolean equals(java.lang.Object o) { return Objects.equals(this.type, configFile.type) && Objects.equals(this.destFile, configFile.destFile) && Objects.equals(this.srcFile, configFile.srcFile) + && Objects.equals(this.visibility, configFile.visibility) && Objects.equals(this.properties, configFile.properties); } @Override public int hashCode() { - return Objects.hash(type, destFile, srcFile, properties); + return Objects.hash(type, destFile, srcFile, visibility, properties); } @Override @@ -217,6 +241,8 @@ public String toString() { .append(" destFile: ").append(toIndentedString(destFile)) .append("\n") .append(" srcFile: ").append(toIndentedString(srcFile)).append("\n") + .append(" visibility: ").append(toIndentedString(visibility)) + .append("\n") .append(" properties: ").append(toIndentedString(properties)) .append("\n") .append("}"); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/ServiceClient.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/ServiceClient.java index 1276022..46bfa7a 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/ServiceClient.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/ServiceClient.java @@ -817,6 +817,21 @@ public int actionDestroy(String serviceName) throws YarnException, + appDir); ret = EXIT_NOT_FOUND; } + + // Delete Public Resource Dir + Path publicResourceDir = new Path(fs.getBasePath(), serviceName); + if (fileSystem.exists(publicResourceDir)) { + if (fileSystem.delete(publicResourceDir, true)) { + LOG.info("Successfully deleted public resource dir for " + + serviceName + ": " + publicResourceDir); + } else { + String message = "Failed to delete public resource dir for service " + + serviceName + " at: " + publicResourceDir; + LOG.info(message); + throw new YarnException(message); + } + } + try { deleteZKNode(serviceName); // don't set destroySucceed to false if no ZK node exists because not @@ -1315,7 +1330,8 @@ private boolean addAMLog4jResource(String serviceName, Configuration conf, new Path(remoteConfPath, YarnServiceConstants.YARN_SERVICE_LOG4J_FILENAME); copy(conf, localFilePath, remoteFilePath); LocalResource localResource = - fs.createAmResource(remoteConfPath, LocalResourceType.FILE); + fs.createAmResource(remoteConfPath, LocalResourceType.FILE, + LocalResourceVisibility.APPLICATION); localResources.put(localFilePath.getName(), localResource); hasAMLog4j = true; } else { @@ -1465,7 +1481,7 @@ private void addKeytabResourceIfSecure(SliderFileSystem fileSystem, return; } LocalResource keytabRes = fileSystem.createAmResource(keytabOnhdfs, - LocalResourceType.FILE); + LocalResourceType.FILE, LocalResourceVisibility.PRIVATE); localResource.put(String.format(YarnServiceConstants.KEYTAB_LOCATION, service.getName()), keytabRes); LOG.info("Adding " + service.getName() + "'s keytab for " diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/conf/YarnServiceConstants.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/conf/YarnServiceConstants.java index 05135fe..dd94065 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/conf/YarnServiceConstants.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/conf/YarnServiceConstants.java @@ -47,6 +47,8 @@ String SERVICES_DIRECTORY = "services"; + String SERVICES_PUBLIC_DIRECTORY = "/tmp/hadoop-yarn/staging/"; + /** * JVM property to define the service lib directory; * this is set by the yarn.sh script diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/provider/ProviderUtils.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/provider/ProviderUtils.java index 5fc96a0..0b091e2 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/provider/ProviderUtils.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/provider/ProviderUtils.java @@ -27,6 +27,7 @@ import org.apache.hadoop.yarn.api.records.Container; import org.apache.hadoop.yarn.api.records.LocalResource; import org.apache.hadoop.yarn.api.records.LocalResourceType; +import org.apache.hadoop.yarn.api.records.LocalResourceVisibility; import org.apache.hadoop.yarn.service.ServiceContext; import org.apache.hadoop.yarn.service.api.records.ConfigFile; import org.apache.hadoop.yarn.service.api.records.ConfigFormat; @@ -191,6 +192,17 @@ public static Path initCompInstanceDir(SliderFileSystem fs, return compInstanceDir; } + public static Path initCompPublicResourceDir(SliderFileSystem fs, + ContainerLaunchService.ComponentLaunchContext compLaunchContext, + ComponentInstance instance) { + Path compDir = fs.getComponentPublicResourceDir( + compLaunchContext.getServiceVersion(), compLaunchContext.getName()); + Path compPublicResourceDir = new Path(compDir, + instance.getCompInstanceName()); + return compPublicResourceDir; + } + + // 1. Create all config files for a component on hdfs for localization // 2. Add the config file to localResource public static synchronized void createConfigFileAndAddLocalResource( @@ -212,6 +224,20 @@ public static synchronized void createConfigFileAndAddLocalResource( log.info("Component instance conf dir already exists: " + compInstanceDir); } + Path compPublicResourceDir = initCompPublicResourceDir(fs, + compLaunchContext, instance); + if (!fs.getFileSystem().exists(compPublicResourceDir)) { + log.info("{} version {} : Creating Public Resource dir on hdfs: {}", + instance.getCompInstanceId(), compLaunchContext.getServiceVersion(), + compPublicResourceDir); + fs.getFileSystem().mkdirs(compPublicResourceDir, + new FsPermission(FsAction.ALL, FsAction.READ_EXECUTE, + FsAction.EXECUTE)); + } else { + log.info("Component instance public resource dir already exists: " + + compPublicResourceDir); + } + log.debug("Tokens substitution for component instance: {}{}{}" + instance .getCompInstanceName(), System.lineSeparator(), tokensForSubstitution); @@ -236,7 +262,14 @@ public static synchronized void createConfigFileAndAddLocalResource( * substitution and merges in new configs, and writes a new file to * compInstanceDir/fileName. */ - Path remoteFile = new Path(compInstanceDir, fileName); + Path remoteFile = null; + LocalResourceVisibility visibility = configFile.getVisibility(); + if (visibility != null && + visibility.equals(LocalResourceVisibility.PUBLIC)) { + remoteFile = new Path(compPublicResourceDir, fileName); + } else { + remoteFile = new Path(compInstanceDir, fileName); + } if (!fs.getFileSystem().exists(remoteFile)) { log.info("Saving config file on hdfs for component " + instance @@ -268,7 +301,8 @@ public static synchronized void createConfigFileAndAddLocalResource( // Add resource for localization LocalResource configResource = - fs.createAmResource(remoteFile, LocalResourceType.FILE); + fs.createAmResource(remoteFile, LocalResourceType.FILE, + configFile.getVisibility()); Path destFile = new Path(configFile.getDestFile()); String symlink = APP_CONF_DIR + "/" + fileName; addLocalResource(launcher, symlink, configResource, destFile, @@ -311,7 +345,8 @@ public static synchronized void handleStaticFilesForLocalization( LocalResource localResource = fs.createAmResource(sourceFile, (staticFile.getType() == ConfigFile.TypeEnum.ARCHIVE ? LocalResourceType.ARCHIVE : - LocalResourceType.FILE)); + LocalResourceType.FILE), staticFile.getVisibility()); + Path destFile = new Path(sourceFile.getName()); if (staticFile.getDestFile() != null && !staticFile.getDestFile() .isEmpty()) { diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/provider/tarball/TarballProviderService.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/provider/tarball/TarballProviderService.java index 87406f7..cd783e7 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/provider/tarball/TarballProviderService.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/provider/tarball/TarballProviderService.java @@ -20,6 +20,7 @@ import org.apache.hadoop.fs.Path; import org.apache.hadoop.yarn.api.records.LocalResource; import org.apache.hadoop.yarn.api.records.LocalResourceType; +import org.apache.hadoop.yarn.api.records.LocalResourceVisibility; import org.apache.hadoop.yarn.service.api.records.Service; import org.apache.hadoop.yarn.service.component.instance.ComponentInstance; import org.apache.hadoop.yarn.service.containerlaunch.ContainerLaunchService; @@ -43,7 +44,8 @@ public void processArtifact(AbstractLauncher launcher, } log.info("Adding resource {}", artifact); LocalResourceType type = LocalResourceType.ARCHIVE; - LocalResource packageResource = fileSystem.createAmResource(artifact, type); + LocalResource packageResource = fileSystem.createAmResource(artifact, type, + LocalResourceVisibility.APPLICATION); launcher.addLocalResource(APP_LIB_DIR, packageResource); } } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/utils/CoreFileSystem.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/utils/CoreFileSystem.java index b9a4649..0ee8e83 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/utils/CoreFileSystem.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/utils/CoreFileSystem.java @@ -384,13 +384,19 @@ public Path getHomeDirectory() { * @param resourceType resource type * @return the local resource for AM */ - public LocalResource createAmResource(Path destPath, LocalResourceType resourceType) throws IOException { + public LocalResource createAmResource(Path destPath, + LocalResourceType resourceType, + LocalResourceVisibility visibility) throws IOException { + FileStatus destStatus = fileSystem.getFileStatus(destPath); LocalResource amResource = Records.newRecord(LocalResource.class); amResource.setType(resourceType); // Set visibility of the resource // Setting to most private option - amResource.setVisibility(LocalResourceVisibility.APPLICATION); + if (visibility == null) { + visibility = LocalResourceVisibility.APPLICATION; + } + amResource.setVisibility(visibility); // Set the resource to be copied over amResource.setResource( URL.fromPath(fileSystem.resolvePath(destStatus.getPath()))); @@ -419,7 +425,7 @@ public LocalResource createAmResource(Path destPath, LocalResourceType resourceT for (FileStatus entry : fileset) { LocalResource resource = createAmResource(entry.getPath(), - LocalResourceType.FILE); + LocalResourceType.FILE, LocalResourceVisibility.APPLICATION); String relativePath = destRelativeDir + "/" + entry.getPath().getName(); localResources.put(relativePath, resource); } @@ -465,7 +471,8 @@ public LocalResource submitFile(File localFile, Path tempPath, String subdir, St // Set the type of resource - file or archive // archives are untarred at destination // we don't need the jar file to be untarred for now - return createAmResource(destPath, LocalResourceType.FILE); + return createAmResource(destPath, LocalResourceType.FILE, + LocalResourceVisibility.APPLICATION); } /** @@ -483,7 +490,7 @@ public void submitTarGzipAndUpdate( BadClusterStateException { Path dependencyLibTarGzip = getDependencyTarGzip(); LocalResource lc = createAmResource(dependencyLibTarGzip, - LocalResourceType.ARCHIVE); + LocalResourceType.ARCHIVE, LocalResourceVisibility.APPLICATION); providerResources.put(YarnServiceConstants.DEPENDENCY_LOCALIZED_DIR_LINK, lc); } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/utils/SliderFileSystem.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/utils/SliderFileSystem.java index c776476..4af9750 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/utils/SliderFileSystem.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/utils/SliderFileSystem.java @@ -21,6 +21,7 @@ import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.Path; +import org.apache.hadoop.yarn.service.conf.YarnServiceConstants; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -63,6 +64,26 @@ public Path getComponentDir(String serviceVersion, String compName) { serviceVersion + "/" + compName); } + public Path getBasePath() { + String tmpDir = configuration.get("hadoop.tmp.dir"); + String basePath = YarnServiceConstants.SERVICE_BASE_DIRECTORY + + "/" + YarnServiceConstants.SERVICES_DIRECTORY; + return new Path(tmpDir, basePath); + } + + /** + * Returns the component public resource directory path. + * + * @param serviceVersion service version + * @param compName component name + * @return component public resource directory + */ + public Path getComponentPublicResourceDir(String serviceVersion, + String compName) { + return new Path(new Path(getBasePath(), getAppDir().getName() + "/" + + "components"), serviceVersion + "/" + compName); + } + /** * Deletes the component directory. * @@ -77,6 +98,12 @@ public void deleteComponentDir(String serviceVersion, String compName) fileSystem.delete(path, true); LOG.debug("deleted dir {}", path); } + Path publicResourceDir = getComponentPublicResourceDir(serviceVersion, + compName); + if (fileSystem.exists(publicResourceDir)) { + fileSystem.delete(publicResourceDir, true); + LOG.debug("deleted public resource dir {}", publicResourceDir); + } } /** @@ -92,6 +119,13 @@ public void deleteComponentsVersionDirIfEmpty(String serviceVersion) fileSystem.delete(path, true); LOG.info("deleted dir {}", path); } + Path publicResourceDir = new Path(new Path(getBasePath(), + getAppDir().getName() + "/" + "components"), serviceVersion); + if (fileSystem.exists(publicResourceDir) + && fileSystem.listStatus(publicResourceDir).length == 0) { + fileSystem.delete(publicResourceDir, true); + LOG.info("deleted public resource dir {}", publicResourceDir); + } } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/test/java/org/apache/hadoop/yarn/service/provider/TestProviderUtils.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/test/java/org/apache/hadoop/yarn/service/provider/TestProviderUtils.java index 84c3b6e..bfdcccd 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/test/java/org/apache/hadoop/yarn/service/provider/TestProviderUtils.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/test/java/org/apache/hadoop/yarn/service/provider/TestProviderUtils.java @@ -63,95 +63,100 @@ public void testStaticFileLocalization() throws IOException { List configFileList = new ArrayList<>(); when(conf.getFiles()).thenReturn(configFileList); when(compLaunchCtx.getConfiguration()).thenReturn(conf); - when(sfs.createAmResource(any(Path.class), any(LocalResourceType.class))) - .thenAnswer(invocationOnMock -> new LocalResource() { - @Override - public URL getResource() { - return URL.fromPath(((Path) invocationOnMock.getArguments()[0])); - } + when(sfs.createAmResource(any(Path.class), any(LocalResourceType.class), + any(LocalResourceVisibility.class))).thenAnswer( + invocationOnMock -> new LocalResource() { + @Override + public URL getResource() { + return URL.fromPath(((Path) invocationOnMock.getArguments()[0])); + } - @Override - public void setResource(URL resource) { + @Override + public void setResource(URL resource) { - } + } - @Override - public long getSize() { - return 0; - } + @Override + public long getSize() { + return 0; + } - @Override - public void setSize(long size) { + @Override + public void setSize(long size) { - } + } - @Override - public long getTimestamp() { - return 0; - } + @Override + public long getTimestamp() { + return 0; + } - @Override - public void setTimestamp(long timestamp) { + @Override + public void setTimestamp(long timestamp) { - } + } - @Override - public LocalResourceType getType() { - return (LocalResourceType) invocationOnMock.getArguments()[1]; - } + @Override + public LocalResourceType getType() { + return (LocalResourceType) invocationOnMock.getArguments()[1]; + } - @Override - public void setType(LocalResourceType type) { + @Override + public void setType(LocalResourceType type) { - } + } - @Override - public LocalResourceVisibility getVisibility() { - return null; - } + @Override + public LocalResourceVisibility getVisibility() { + return LocalResourceVisibility.APPLICATION; + } - @Override - public void setVisibility(LocalResourceVisibility visibility) { + @Override + public void setVisibility(LocalResourceVisibility visibility) { - } + } - @Override - public String getPattern() { - return null; - } + @Override + public String getPattern() { + return null; + } - @Override - public void setPattern(String pattern) { + @Override + public void setPattern(String pattern) { - } + } - @Override - public boolean getShouldBeUploadedToSharedCache() { - return false; - } + @Override + public boolean getShouldBeUploadedToSharedCache() { + return false; + } - @Override - public void setShouldBeUploadedToSharedCache( - boolean shouldBeUploadedToSharedCache) { + @Override + public void setShouldBeUploadedToSharedCache( + boolean shouldBeUploadedToSharedCache) { - } - }); + } + }); // Initialize list of files. //archive configFileList.add(new ConfigFile().srcFile("hdfs://default/sourceFile1") - .destFile("destFile1").type(ConfigFile.TypeEnum.ARCHIVE)); + .destFile("destFile1").type(ConfigFile.TypeEnum.ARCHIVE) + .visibility(LocalResourceVisibility.APPLICATION)); //static file configFileList.add(new ConfigFile().srcFile("hdfs://default/sourceFile2") - .destFile("folder/destFile_2").type(ConfigFile.TypeEnum.STATIC)); + .destFile("folder/destFile_2").type(ConfigFile.TypeEnum.STATIC) + .visibility(LocalResourceVisibility.APPLICATION)); //This will be ignored since type is JSON configFileList.add(new ConfigFile().srcFile("hdfs://default/sourceFile3") - .destFile("destFile3").type(ConfigFile.TypeEnum.JSON)); + .destFile("destFile3").type(ConfigFile.TypeEnum.JSON) + .visibility(LocalResourceVisibility.APPLICATION)); //No destination file specified configFileList.add(new ConfigFile().srcFile("hdfs://default/sourceFile4") - .type(ConfigFile.TypeEnum.STATIC)); + .type(ConfigFile.TypeEnum.STATIC) + .visibility(LocalResourceVisibility.APPLICATION)); ProviderService.ResolvedLaunchParams resolved = new ProviderService.ResolvedLaunchParams(); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/DockerLinuxContainerRuntime.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/DockerLinuxContainerRuntime.java index 50721de..dce2490 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/DockerLinuxContainerRuntime.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/DockerLinuxContainerRuntime.java @@ -235,6 +235,9 @@ @InterfaceAudience.Private public static final String ENV_DOCKER_CONTAINER_DOCKER_RUNTIME = "YARN_CONTAINER_RUNTIME_DOCKER_RUNTIME"; + @InterfaceAudience.Private + public static final String ENV_DOCKER_CONTAINER_DOCKER_SERVICE_MODE = + "YARN_CONTAINER_RUNTIME_DOCKER_SERVICE_MODE"; @InterfaceAudience.Private private static final String RUNTIME_TYPE = "DOCKER"; @@ -588,7 +591,9 @@ public void launchContainer(ContainerRuntimeContext ctx) String network = environment.get(ENV_DOCKER_CONTAINER_NETWORK); String hostname = environment.get(ENV_DOCKER_CONTAINER_HOSTNAME); String runtime = environment.get(ENV_DOCKER_CONTAINER_DOCKER_RUNTIME); - boolean useEntryPoint = checkUseEntryPoint(environment); + boolean serviceMode = Boolean.parseBoolean(environment.get( + ENV_DOCKER_CONTAINER_DOCKER_SERVICE_MODE)); + boolean useEntryPoint = serviceMode || checkUseEntryPoint(environment); if (imageName == null || imageName.isEmpty()) { imageName = defaultImageName; @@ -679,10 +684,12 @@ public void launchContainer(ContainerRuntimeContext ctx) runCommand.addRuntime(runtime); } - runCommand.addAllReadWriteMountLocations(containerLogDirs); - runCommand.addAllReadWriteMountLocations(applicationLocalDirs); - runCommand.addAllReadOnlyMountLocations(filecacheDirs); - runCommand.addAllReadOnlyMountLocations(userFilecacheDirs); + if (!serviceMode) { + runCommand.addAllReadWriteMountLocations(containerLogDirs); + runCommand.addAllReadWriteMountLocations(applicationLocalDirs); + runCommand.addAllReadOnlyMountLocations(filecacheDirs); + runCommand.addAllReadOnlyMountLocations(userFilecacheDirs); + } if (environment.containsKey(ENV_DOCKER_CONTAINER_MOUNTS)) { Matcher parsedMounts = USER_MOUNT_PATTERN.matcher( @@ -800,11 +807,20 @@ public void launchContainer(ContainerRuntimeContext ctx) runCommand.setYarnSysFS(true); } + // In service mode, the YARN log dirs are not mounted into the container. + // As a result, the container fails to start due to stdout and stderr output + // being sent to a file in a directory that does not exist. In service mode, + // only supply the command with no stdout or stderr redirection. + List commands = container.getLaunchContext().getCommands(); + if (serviceMode) { + commands = Arrays.asList( + String.join(" ", commands).split("1>")[0].split(" ")); + } + if (useEntryPoint) { runCommand.setOverrideDisabled(true); runCommand.addEnv(environment); - runCommand.setOverrideCommandWithArgs(container.getLaunchContext() - .getCommands()); + runCommand.setOverrideCommandWithArgs(commands); runCommand.disableDetach(); runCommand.setLogDir(container.getLogDir()); } else { @@ -818,6 +834,10 @@ public void launchContainer(ContainerRuntimeContext ctx) runCommand.detachOnRun(); } + if (serviceMode) { + runCommand.setServiceMode(serviceMode); + } + if(enableUserReMapping) { if (!allowPrivilegedContainerExecution(container)) { runCommand.groupAdd(groups); @@ -1279,11 +1299,14 @@ private void handleContainerKill(ContainerRuntimeContext ctx, throw new ContainerExecutionException(e); } + boolean serviceMode = Boolean.parseBoolean(env.get( + ENV_DOCKER_CONTAINER_DOCKER_SERVICE_MODE)); + // Only need to check whether the container was asked to be privileged. // If the container had failed the permissions checks upon launch, it // would have never been launched and thus we wouldn't be here // attempting to signal it. - if (isContainerRequestedAsPrivileged(container)) { + if (isContainerRequestedAsPrivileged(container) || serviceMode) { String containerId = container.getContainerId().toString(); DockerCommandExecutor.DockerContainerStatus containerStatus = DockerCommandExecutor.getContainerStatus(containerId, diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/docker/DockerRunCommand.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/docker/DockerRunCommand.java index b0603a3..7fb0e40 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/docker/DockerRunCommand.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/docker/DockerRunCommand.java @@ -199,6 +199,12 @@ public DockerRunCommand setLogDir(String logDir) { return this; } + public DockerRunCommand setServiceMode(boolean serviceMode) { + String value = Boolean.toString(serviceMode); + super.addCommandArguments("service-mode", value); + return this; + } + /** * Check if user defined environment variables are empty. * diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/container-executor.h b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/container-executor.h index b215af7..757bd16 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/container-executor.h +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/container-executor.h @@ -325,12 +325,6 @@ int sync_yarn_sysfs(char* const* local_dirs, const char *running_user, */ int execute_regex_match(const char *regex_str, const char *input); -/** - * Validate the docker image name matches the expected input. - * Return 0 on success. - */ -int validate_docker_image_name(const char *image_name); - struct configuration* get_cfg(); /** diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/utils/docker-util.c b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/utils/docker-util.c index 1711433..3ef571f 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/utils/docker-util.c +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/utils/docker-util.c @@ -28,6 +28,7 @@ #include "docker-util.h" #include "string-utils.h" #include "util.h" +#include "container-executor.h" #include #include #include @@ -374,6 +375,8 @@ const char *get_docker_error_message(const int error_code) { return "Invalid docker tmpfs mount"; case INVALID_DOCKER_RUNTIME: return "Invalid docker runtime"; + case SERVICE_MODE_DISABLED: + return "Service mode disabled"; default: return "Unknown error"; } @@ -987,6 +990,22 @@ static int set_runtime(const struct configuration *command_config, return ret; } +int is_service_mode_enabled(const struct configuration *command_config, + const struct configuration *executor_cfg, args *args) { + int ret = 0; + struct section *section = get_configuration_section(CONTAINER_EXECUTOR_CFG_DOCKER_SECTION, executor_cfg); + char *value = get_configuration_value("service-mode", DOCKER_COMMAND_FILE_SECTION, command_config); + if (value != NULL && strcasecmp(value, "true") == 0) { + if (is_feature_enabled(DOCKER_SERVICE_MODE_ENABLED_KEY, ret, section)) { + ret = 1; + } else { + ret = SERVICE_MODE_DISABLED; + } + } + free(value); + return ret; +} + static int add_ports_mapping_to_command(const struct configuration *command_config, args *args) { int i = 0, ret = 0; char *network_type = (char*) malloc(128); @@ -1595,12 +1614,19 @@ int get_docker_run_command(const char *command_file, const struct configuration char *privileged = NULL; char *no_new_privileges_enabled = NULL; char *use_entry_point = NULL; + int service_mode_enabled = 0; struct configuration command_config = {0, NULL}; ret = read_and_verify_command_file(command_file, DOCKER_RUN_COMMAND, &command_config); if (ret != 0) { goto free_and_exit; } + service_mode_enabled = is_service_mode_enabled(&command_config, conf, args); + if (service_mode_enabled == SERVICE_MODE_DISABLED) { + ret = SERVICE_MODE_DISABLED; + goto free_and_exit; + } + use_entry_point = get_configuration_value("use-entry-point", DOCKER_COMMAND_FILE_SECTION, &command_config); if (use_entry_point != NULL && strcasecmp(use_entry_point, "true") == 0) { entry_point = 1; @@ -1612,10 +1638,13 @@ int get_docker_run_command(const char *command_file, const struct configuration ret = INVALID_DOCKER_CONTAINER_NAME; goto free_and_exit; } - user = get_configuration_value("user", DOCKER_COMMAND_FILE_SECTION, &command_config); - if (user == NULL) { - ret = INVALID_DOCKER_USER_NAME; - goto free_and_exit; + + if (!service_mode_enabled) { + user = get_configuration_value("user", DOCKER_COMMAND_FILE_SECTION, &command_config); + if (user == NULL) { + ret = INVALID_DOCKER_USER_NAME; + goto free_and_exit; + } } image = get_configuration_value("image", DOCKER_COMMAND_FILE_SECTION, &command_config); if (image == NULL || validate_docker_image_name(image) != 0) { @@ -1640,12 +1669,14 @@ int get_docker_run_command(const char *command_file, const struct configuration privileged = get_configuration_value("privileged", DOCKER_COMMAND_FILE_SECTION, &command_config); if (privileged == NULL || strcmp(privileged, "false") == 0) { - char *user_buffer = make_string("--user=%s", user); - ret = add_to_args(args, user_buffer); - free(user_buffer); - if (ret != 0) { - ret = BUFFER_TOO_SMALL; - goto free_and_exit; + if (!service_mode_enabled) { + char *user_buffer = make_string("--user=%s", user); + ret = add_to_args(args, user_buffer); + free(user_buffer); + if (ret != 0) { + ret = BUFFER_TOO_SMALL; + goto free_and_exit; + } } no_new_privileges_enabled = get_configuration_value("docker.no-new-privileges.enabled", @@ -1725,9 +1756,11 @@ int get_docker_run_command(const char *command_file, const struct configuration goto free_and_exit; } - ret = set_group_add(&command_config, args); - if (ret != 0) { - goto free_and_exit; + if (!service_mode_enabled) { + ret = set_group_add(&command_config, args); + if (ret != 0) { + goto free_and_exit; + } } ret = set_devices(&command_config, conf, args); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/utils/docker-util.h b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/utils/docker-util.h index 07da195..d9d34a0 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/utils/docker-util.h +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/utils/docker-util.h @@ -36,6 +36,7 @@ #define DOCKER_START_COMMAND "start" #define DOCKER_EXEC_COMMAND "exec" #define DOCKER_IMAGES_COMMAND "images" +#define DOCKER_SERVICE_MODE_ENABLED_KEY "docker.service-mode.enabled" #define DOCKER_ARG_MAX 1024 #define ARGS_INITIAL_VALUE { 0 }; @@ -71,7 +72,8 @@ enum docker_error_codes { INVALID_PID_NAMESPACE, INVALID_DOCKER_IMAGE_TRUST, INVALID_DOCKER_TMPFS_MOUNT, - INVALID_DOCKER_RUNTIME + INVALID_DOCKER_RUNTIME, + SERVICE_MODE_DISABLED }; /** diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/DockerContainers.md b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/DockerContainers.md index e30ac98..db9c56d 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/DockerContainers.md +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/DockerContainers.md @@ -285,6 +285,7 @@ are allowed. It contains the following properties: | `docker.inspect.max.retries` | Integer value to check docker container readiness. Each inspection is set with 3 seconds delay. Default value of 10 will wait 30 seconds for docker container to become ready before marked as container failed. | | `docker.no-new-privileges.enabled` | Enable/disable the no-new-privileges flag for docker run. Set to "true" to enable, disabled by default. | | `docker.allowed.runtimes` | Comma seperated runtimes that containers are allowed to use. By default no runtimes are allowed to be added.| +| `docker.service-mode.enabled` | Set to "true" or "false" to enable or disable docker container service mode. Default value is "false". | Please note that if you wish to run Docker containers that require access to the YARN local directories, you must add them to the docker.allowed.rw-mounts list. @@ -436,6 +437,7 @@ environment variables in the application's environment: | `YARN_CONTAINER_RUNTIME_DOCKER_TMPFS_MOUNTS` | Adds additional tmpfs mounts to the Docker container. The value of the environment variable should be a comma-separated list of absolute mount points within the container. | | `YARN_CONTAINER_RUNTIME_DOCKER_DELAYED_REMOVAL` | Allows a user to request delayed deletion of the Docker container on a per container basis. If true, Docker containers will not be removed until the duration defined by yarn.nodemanager.delete.debug-delay-sec has elapsed. Administrators can disable this feature through the yarn-site property yarn.nodemanager.runtime.linux.docker.delayed-removal.allowed. This feature is disabled by default. When this feature is disabled or set to false, the container will be removed as soon as it exits. | | `YARN_CONTAINER_RUNTIME_YARN_SYSFS_ENABLE` | Enable mounting of container working directory sysfs sub-directory into Docker container /hadoop/yarn/sysfs. This is useful for populating cluster information into container. | +| `YARN_CONTAINER_RUNTIME_DOCKER_SERVICE_MODE` | Enable Service Mode which runs the docker container as defined by the image but does not set the user (--user and --group-add). | The first two are required. The remainder can be set as needed. While controlling the container type through environment variables is somewhat less @@ -1080,3 +1082,24 @@ YARN service framework automatically populates cluster information to /hadoop/yarn/sysfs/app.json. For more information about YARN service, see: [YARN Service](./yarn-service/Overview.html). +Docker Container Service Mode +----------------------------- + +Docker Container Service Mode runs the container as defined by the image +but does not set the user (--user and --group-add). This mode is disabled +by default. The administrator sets docker.service-mode.enabled to true +in container-executor.cfg under docker section to enable. + +Part of a container-executor.cfg which allows docker service mode is below: + +``` +yarn.nodemanager.linux-container-executor.group=yarn +[docker] + module.enabled=true + docker.privileged-containers.enabled=true + docker.service-mode.enabled=true +``` + +Application User can enable or disable service mode at job level by exporting +environment variable YARN_CONTAINER_RUNTIME_DOCKER_SERVICE_MODE in the application's +environment with value true or false respectively. -- 2.7.4 (Apple Git-66)