From eb3c42de4da5602dc2597c3f619e47b6da16f6cd Mon Sep 17 00:00:00 2001 From: Prabhu Joseph Date: Fri, 4 Oct 2019 02:06:47 +0530 Subject: [PATCH] YARN-9860. Enable service mode for Docker containers on YARN. 1. This introduces service mode for Docker container which allows injecting configuration. 2. This adds new field visibility for files to be localized in YARN Native Service. --- .../yarn/service/api/records/ConfigFile.java | 28 +++++++++- .../hadoop/yarn/service/client/ServiceClient.java | 21 +++++++- .../yarn/service/conf/YarnServiceConstants.java | 2 + .../yarn/service/provider/ProviderUtils.java | 41 +++++++++++++-- .../provider/tarball/TarballProviderService.java | 4 +- .../hadoop/yarn/service/utils/CoreFileSystem.java | 17 +++++-- .../yarn/service/utils/SliderFileSystem.java | 36 +++++++++++++ .../yarn/service/provider/TestProviderUtils.java | 19 ++++--- .../linux/runtime/DockerLinuxContainerRuntime.java | 39 +++++++++++--- .../linux/runtime/docker/DockerRunCommand.java | 6 +++ .../container-executor/impl/container-executor.h | 6 --- .../container-executor/impl/utils/docker-util.c | 59 +++++++++++++++++----- .../container-executor/impl/utils/docker-util.h | 4 +- 13 files changed, 235 insertions(+), 47 deletions(-) diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/api/records/ConfigFile.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/api/records/ConfigFile.java index c09373f..060e204 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/api/records/ConfigFile.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/api/records/ConfigFile.java @@ -24,6 +24,7 @@ import io.swagger.annotations.ApiModelProperty; import org.apache.hadoop.classification.InterfaceAudience; import org.apache.hadoop.classification.InterfaceStability; +import org.apache.hadoop.yarn.api.records.LocalResourceVisibility; import javax.xml.bind.annotation.XmlElement; import javax.xml.bind.annotation.XmlEnum; @@ -73,6 +74,7 @@ public String toString() { private TypeEnum type = null; private String destFile = null; private String srcFile = null; + private LocalResourceVisibility visibility = null; private Map properties = new HashMap<>(); public ConfigFile copy() { @@ -80,6 +82,7 @@ public ConfigFile copy() { copy.setType(this.getType()); copy.setSrcFile(this.getSrcFile()); copy.setDestFile(this.getDestFile()); + copy.setVisibility(this.visibility); if (this.getProperties() != null && !this.getProperties().isEmpty()) { copy.getProperties().putAll(this.getProperties()); } @@ -150,6 +153,26 @@ public void setSrcFile(String srcFile) { this.srcFile = srcFile; } + + /** + * Visibility of the Config file. + **/ + public ConfigFile visibility(LocalResourceVisibility localrsrcVisibility) { + this.visibility = localrsrcVisibility; + return this; + } + + @ApiModelProperty(example = "null", value = "Visibility of the Config file") + @JsonProperty("visibility") + public LocalResourceVisibility getVisibility() { + return visibility; + } + + @XmlElement(name = "visibility", defaultValue="APPLICATION") + public void setVisibility(LocalResourceVisibility localrsrcVisibility) { + this.visibility = localrsrcVisibility; + } + /** A blob of key value pairs that will be dumped in the dest_file in the format as specified in type. If src_file is specified, src_file content are dumped @@ -200,12 +223,13 @@ public boolean equals(java.lang.Object o) { return Objects.equals(this.type, configFile.type) && Objects.equals(this.destFile, configFile.destFile) && Objects.equals(this.srcFile, configFile.srcFile) + && Objects.equals(this.visibility, configFile.visibility) && Objects.equals(this.properties, configFile.properties); } @Override public int hashCode() { - return Objects.hash(type, destFile, srcFile, properties); + return Objects.hash(type, destFile, srcFile, visibility, properties); } @Override @@ -217,6 +241,8 @@ public String toString() { .append(" destFile: ").append(toIndentedString(destFile)) .append("\n") .append(" srcFile: ").append(toIndentedString(srcFile)).append("\n") + .append(" visibility: ").append(toIndentedString(visibility)) + .append("\n") .append(" properties: ").append(toIndentedString(properties)) .append("\n") .append("}"); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/ServiceClient.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/ServiceClient.java index 1276022..058946f 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/ServiceClient.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/ServiceClient.java @@ -817,6 +817,22 @@ public int actionDestroy(String serviceName) throws YarnException, + appDir); ret = EXIT_NOT_FOUND; } + + // Delete Public Resource Dir + Path publicResourceDir = new Path(fs.getBasePath( + fs.getHomeDirectory().getName()), serviceName); + if (fileSystem.exists(publicResourceDir)) { + if (fileSystem.delete(publicResourceDir, true)) { + LOG.info("Successfully deleted public resource dir for " + + serviceName + ": " + publicResourceDir); + } else { + String message = "Failed to delete public resource dir for service " + + serviceName + " at: " + publicResourceDir; + LOG.info(message); + throw new YarnException(message); + } + } + try { deleteZKNode(serviceName); // don't set destroySucceed to false if no ZK node exists because not @@ -1315,7 +1331,8 @@ private boolean addAMLog4jResource(String serviceName, Configuration conf, new Path(remoteConfPath, YarnServiceConstants.YARN_SERVICE_LOG4J_FILENAME); copy(conf, localFilePath, remoteFilePath); LocalResource localResource = - fs.createAmResource(remoteConfPath, LocalResourceType.FILE); + fs.createAmResource(remoteConfPath, LocalResourceType.FILE, + LocalResourceVisibility.APPLICATION); localResources.put(localFilePath.getName(), localResource); hasAMLog4j = true; } else { @@ -1465,7 +1482,7 @@ private void addKeytabResourceIfSecure(SliderFileSystem fileSystem, return; } LocalResource keytabRes = fileSystem.createAmResource(keytabOnhdfs, - LocalResourceType.FILE); + LocalResourceType.FILE, LocalResourceVisibility.APPLICATION); localResource.put(String.format(YarnServiceConstants.KEYTAB_LOCATION, service.getName()), keytabRes); LOG.info("Adding " + service.getName() + "'s keytab for " diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/conf/YarnServiceConstants.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/conf/YarnServiceConstants.java index 05135fe..dd94065 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/conf/YarnServiceConstants.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/conf/YarnServiceConstants.java @@ -47,6 +47,8 @@ String SERVICES_DIRECTORY = "services"; + String SERVICES_PUBLIC_DIRECTORY = "/tmp/hadoop-yarn/staging/"; + /** * JVM property to define the service lib directory; * this is set by the yarn.sh script diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/provider/ProviderUtils.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/provider/ProviderUtils.java index 5fc96a0..0b091e2 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/provider/ProviderUtils.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/provider/ProviderUtils.java @@ -27,6 +27,7 @@ import org.apache.hadoop.yarn.api.records.Container; import org.apache.hadoop.yarn.api.records.LocalResource; import org.apache.hadoop.yarn.api.records.LocalResourceType; +import org.apache.hadoop.yarn.api.records.LocalResourceVisibility; import org.apache.hadoop.yarn.service.ServiceContext; import org.apache.hadoop.yarn.service.api.records.ConfigFile; import org.apache.hadoop.yarn.service.api.records.ConfigFormat; @@ -191,6 +192,17 @@ public static Path initCompInstanceDir(SliderFileSystem fs, return compInstanceDir; } + public static Path initCompPublicResourceDir(SliderFileSystem fs, + ContainerLaunchService.ComponentLaunchContext compLaunchContext, + ComponentInstance instance) { + Path compDir = fs.getComponentPublicResourceDir( + compLaunchContext.getServiceVersion(), compLaunchContext.getName()); + Path compPublicResourceDir = new Path(compDir, + instance.getCompInstanceName()); + return compPublicResourceDir; + } + + // 1. Create all config files for a component on hdfs for localization // 2. Add the config file to localResource public static synchronized void createConfigFileAndAddLocalResource( @@ -212,6 +224,20 @@ public static synchronized void createConfigFileAndAddLocalResource( log.info("Component instance conf dir already exists: " + compInstanceDir); } + Path compPublicResourceDir = initCompPublicResourceDir(fs, + compLaunchContext, instance); + if (!fs.getFileSystem().exists(compPublicResourceDir)) { + log.info("{} version {} : Creating Public Resource dir on hdfs: {}", + instance.getCompInstanceId(), compLaunchContext.getServiceVersion(), + compPublicResourceDir); + fs.getFileSystem().mkdirs(compPublicResourceDir, + new FsPermission(FsAction.ALL, FsAction.READ_EXECUTE, + FsAction.EXECUTE)); + } else { + log.info("Component instance public resource dir already exists: " + + compPublicResourceDir); + } + log.debug("Tokens substitution for component instance: {}{}{}" + instance .getCompInstanceName(), System.lineSeparator(), tokensForSubstitution); @@ -236,7 +262,14 @@ public static synchronized void createConfigFileAndAddLocalResource( * substitution and merges in new configs, and writes a new file to * compInstanceDir/fileName. */ - Path remoteFile = new Path(compInstanceDir, fileName); + Path remoteFile = null; + LocalResourceVisibility visibility = configFile.getVisibility(); + if (visibility != null && + visibility.equals(LocalResourceVisibility.PUBLIC)) { + remoteFile = new Path(compPublicResourceDir, fileName); + } else { + remoteFile = new Path(compInstanceDir, fileName); + } if (!fs.getFileSystem().exists(remoteFile)) { log.info("Saving config file on hdfs for component " + instance @@ -268,7 +301,8 @@ public static synchronized void createConfigFileAndAddLocalResource( // Add resource for localization LocalResource configResource = - fs.createAmResource(remoteFile, LocalResourceType.FILE); + fs.createAmResource(remoteFile, LocalResourceType.FILE, + configFile.getVisibility()); Path destFile = new Path(configFile.getDestFile()); String symlink = APP_CONF_DIR + "/" + fileName; addLocalResource(launcher, symlink, configResource, destFile, @@ -311,7 +345,8 @@ public static synchronized void handleStaticFilesForLocalization( LocalResource localResource = fs.createAmResource(sourceFile, (staticFile.getType() == ConfigFile.TypeEnum.ARCHIVE ? LocalResourceType.ARCHIVE : - LocalResourceType.FILE)); + LocalResourceType.FILE), staticFile.getVisibility()); + Path destFile = new Path(sourceFile.getName()); if (staticFile.getDestFile() != null && !staticFile.getDestFile() .isEmpty()) { diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/provider/tarball/TarballProviderService.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/provider/tarball/TarballProviderService.java index 87406f7..cd783e7 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/provider/tarball/TarballProviderService.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/provider/tarball/TarballProviderService.java @@ -20,6 +20,7 @@ import org.apache.hadoop.fs.Path; import org.apache.hadoop.yarn.api.records.LocalResource; import org.apache.hadoop.yarn.api.records.LocalResourceType; +import org.apache.hadoop.yarn.api.records.LocalResourceVisibility; import org.apache.hadoop.yarn.service.api.records.Service; import org.apache.hadoop.yarn.service.component.instance.ComponentInstance; import org.apache.hadoop.yarn.service.containerlaunch.ContainerLaunchService; @@ -43,7 +44,8 @@ public void processArtifact(AbstractLauncher launcher, } log.info("Adding resource {}", artifact); LocalResourceType type = LocalResourceType.ARCHIVE; - LocalResource packageResource = fileSystem.createAmResource(artifact, type); + LocalResource packageResource = fileSystem.createAmResource(artifact, type, + LocalResourceVisibility.APPLICATION); launcher.addLocalResource(APP_LIB_DIR, packageResource); } } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/utils/CoreFileSystem.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/utils/CoreFileSystem.java index b9a4649..0ee8e83 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/utils/CoreFileSystem.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/utils/CoreFileSystem.java @@ -384,13 +384,19 @@ public Path getHomeDirectory() { * @param resourceType resource type * @return the local resource for AM */ - public LocalResource createAmResource(Path destPath, LocalResourceType resourceType) throws IOException { + public LocalResource createAmResource(Path destPath, + LocalResourceType resourceType, + LocalResourceVisibility visibility) throws IOException { + FileStatus destStatus = fileSystem.getFileStatus(destPath); LocalResource amResource = Records.newRecord(LocalResource.class); amResource.setType(resourceType); // Set visibility of the resource // Setting to most private option - amResource.setVisibility(LocalResourceVisibility.APPLICATION); + if (visibility == null) { + visibility = LocalResourceVisibility.APPLICATION; + } + amResource.setVisibility(visibility); // Set the resource to be copied over amResource.setResource( URL.fromPath(fileSystem.resolvePath(destStatus.getPath()))); @@ -419,7 +425,7 @@ public LocalResource createAmResource(Path destPath, LocalResourceType resourceT for (FileStatus entry : fileset) { LocalResource resource = createAmResource(entry.getPath(), - LocalResourceType.FILE); + LocalResourceType.FILE, LocalResourceVisibility.APPLICATION); String relativePath = destRelativeDir + "/" + entry.getPath().getName(); localResources.put(relativePath, resource); } @@ -465,7 +471,8 @@ public LocalResource submitFile(File localFile, Path tempPath, String subdir, St // Set the type of resource - file or archive // archives are untarred at destination // we don't need the jar file to be untarred for now - return createAmResource(destPath, LocalResourceType.FILE); + return createAmResource(destPath, LocalResourceType.FILE, + LocalResourceVisibility.APPLICATION); } /** @@ -483,7 +490,7 @@ public void submitTarGzipAndUpdate( BadClusterStateException { Path dependencyLibTarGzip = getDependencyTarGzip(); LocalResource lc = createAmResource(dependencyLibTarGzip, - LocalResourceType.ARCHIVE); + LocalResourceType.ARCHIVE, LocalResourceVisibility.APPLICATION); providerResources.put(YarnServiceConstants.DEPENDENCY_LOCALIZED_DIR_LINK, lc); } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/utils/SliderFileSystem.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/utils/SliderFileSystem.java index c776476..629adf7 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/utils/SliderFileSystem.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/utils/SliderFileSystem.java @@ -21,6 +21,7 @@ import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.Path; +import org.apache.hadoop.yarn.service.conf.YarnServiceConstants; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -63,6 +64,27 @@ public Path getComponentDir(String serviceVersion, String compName) { serviceVersion + "/" + compName); } + public String getBasePath(String user) { + String basePath = YarnServiceConstants.SERVICES_PUBLIC_DIRECTORY + user + + "/" + YarnServiceConstants.SERVICE_BASE_DIRECTORY + "/" + + YarnServiceConstants.SERVICES_DIRECTORY; + return basePath; + } + + /** + * Returns the component public resource directory path. + * + * @param serviceVersion service version + * @param compName component name + * @return component public resource directory + */ + public Path getComponentPublicResourceDir(String serviceVersion, + String compName) { + return new Path(new Path(getBasePath(System.getenv("USER")), + getAppDir().getName() + "/" + + "components"), serviceVersion + "/" + compName); + } + /** * Deletes the component directory. * @@ -77,6 +99,12 @@ public void deleteComponentDir(String serviceVersion, String compName) fileSystem.delete(path, true); LOG.debug("deleted dir {}", path); } + Path publicResourceDir = getComponentPublicResourceDir(serviceVersion, + compName); + if (fileSystem.exists(publicResourceDir)) { + fileSystem.delete(publicResourceDir, true); + LOG.debug("deleted public resource dir {}", publicResourceDir); + } } /** @@ -92,6 +120,14 @@ public void deleteComponentsVersionDirIfEmpty(String serviceVersion) fileSystem.delete(path, true); LOG.info("deleted dir {}", path); } + Path publicResourceDir = new Path(new Path(getBasePath( + System.getenv("USER")), getAppDir().getName() + + "/" + "components"), serviceVersion); + if (fileSystem.exists(publicResourceDir) + && fileSystem.listStatus(publicResourceDir).length == 0) { + fileSystem.delete(publicResourceDir, true); + LOG.info("deleted public resource dir {}", publicResourceDir); + } } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/test/java/org/apache/hadoop/yarn/service/provider/TestProviderUtils.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/test/java/org/apache/hadoop/yarn/service/provider/TestProviderUtils.java index 84c3b6e..2c85240 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/test/java/org/apache/hadoop/yarn/service/provider/TestProviderUtils.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/test/java/org/apache/hadoop/yarn/service/provider/TestProviderUtils.java @@ -63,8 +63,9 @@ public void testStaticFileLocalization() throws IOException { List configFileList = new ArrayList<>(); when(conf.getFiles()).thenReturn(configFileList); when(compLaunchCtx.getConfiguration()).thenReturn(conf); - when(sfs.createAmResource(any(Path.class), any(LocalResourceType.class))) - .thenAnswer(invocationOnMock -> new LocalResource() { + when(sfs.createAmResource(any(Path.class), any(LocalResourceType.class), + any(LocalResourceVisibility.class))).thenAnswer( + invocationOnMock -> new LocalResource() { @Override public URL getResource() { return URL.fromPath(((Path) invocationOnMock.getArguments()[0])); @@ -107,7 +108,7 @@ public void setType(LocalResourceType type) { @Override public LocalResourceVisibility getVisibility() { - return null; + return LocalResourceVisibility.APPLICATION; } @Override @@ -140,18 +141,22 @@ public void setShouldBeUploadedToSharedCache( // Initialize list of files. //archive configFileList.add(new ConfigFile().srcFile("hdfs://default/sourceFile1") - .destFile("destFile1").type(ConfigFile.TypeEnum.ARCHIVE)); + .destFile("destFile1").type(ConfigFile.TypeEnum.ARCHIVE) + .visibility(LocalResourceVisibility.APPLICATION)); //static file configFileList.add(new ConfigFile().srcFile("hdfs://default/sourceFile2") - .destFile("folder/destFile_2").type(ConfigFile.TypeEnum.STATIC)); + .destFile("folder/destFile_2").type(ConfigFile.TypeEnum.STATIC) + .visibility(LocalResourceVisibility.APPLICATION)); //This will be ignored since type is JSON configFileList.add(new ConfigFile().srcFile("hdfs://default/sourceFile3") - .destFile("destFile3").type(ConfigFile.TypeEnum.JSON)); + .destFile("destFile3").type(ConfigFile.TypeEnum.JSON) + .visibility(LocalResourceVisibility.APPLICATION)); //No destination file specified configFileList.add(new ConfigFile().srcFile("hdfs://default/sourceFile4") - .type(ConfigFile.TypeEnum.STATIC)); + .type(ConfigFile.TypeEnum.STATIC) + .visibility(LocalResourceVisibility.APPLICATION)); ProviderService.ResolvedLaunchParams resolved = new ProviderService.ResolvedLaunchParams(); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/DockerLinuxContainerRuntime.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/DockerLinuxContainerRuntime.java index 50721de..dce2490 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/DockerLinuxContainerRuntime.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/DockerLinuxContainerRuntime.java @@ -235,6 +235,9 @@ @InterfaceAudience.Private public static final String ENV_DOCKER_CONTAINER_DOCKER_RUNTIME = "YARN_CONTAINER_RUNTIME_DOCKER_RUNTIME"; + @InterfaceAudience.Private + public static final String ENV_DOCKER_CONTAINER_DOCKER_SERVICE_MODE = + "YARN_CONTAINER_RUNTIME_DOCKER_SERVICE_MODE"; @InterfaceAudience.Private private static final String RUNTIME_TYPE = "DOCKER"; @@ -588,7 +591,9 @@ public void launchContainer(ContainerRuntimeContext ctx) String network = environment.get(ENV_DOCKER_CONTAINER_NETWORK); String hostname = environment.get(ENV_DOCKER_CONTAINER_HOSTNAME); String runtime = environment.get(ENV_DOCKER_CONTAINER_DOCKER_RUNTIME); - boolean useEntryPoint = checkUseEntryPoint(environment); + boolean serviceMode = Boolean.parseBoolean(environment.get( + ENV_DOCKER_CONTAINER_DOCKER_SERVICE_MODE)); + boolean useEntryPoint = serviceMode || checkUseEntryPoint(environment); if (imageName == null || imageName.isEmpty()) { imageName = defaultImageName; @@ -679,10 +684,12 @@ public void launchContainer(ContainerRuntimeContext ctx) runCommand.addRuntime(runtime); } - runCommand.addAllReadWriteMountLocations(containerLogDirs); - runCommand.addAllReadWriteMountLocations(applicationLocalDirs); - runCommand.addAllReadOnlyMountLocations(filecacheDirs); - runCommand.addAllReadOnlyMountLocations(userFilecacheDirs); + if (!serviceMode) { + runCommand.addAllReadWriteMountLocations(containerLogDirs); + runCommand.addAllReadWriteMountLocations(applicationLocalDirs); + runCommand.addAllReadOnlyMountLocations(filecacheDirs); + runCommand.addAllReadOnlyMountLocations(userFilecacheDirs); + } if (environment.containsKey(ENV_DOCKER_CONTAINER_MOUNTS)) { Matcher parsedMounts = USER_MOUNT_PATTERN.matcher( @@ -800,11 +807,20 @@ public void launchContainer(ContainerRuntimeContext ctx) runCommand.setYarnSysFS(true); } + // In service mode, the YARN log dirs are not mounted into the container. + // As a result, the container fails to start due to stdout and stderr output + // being sent to a file in a directory that does not exist. In service mode, + // only supply the command with no stdout or stderr redirection. + List commands = container.getLaunchContext().getCommands(); + if (serviceMode) { + commands = Arrays.asList( + String.join(" ", commands).split("1>")[0].split(" ")); + } + if (useEntryPoint) { runCommand.setOverrideDisabled(true); runCommand.addEnv(environment); - runCommand.setOverrideCommandWithArgs(container.getLaunchContext() - .getCommands()); + runCommand.setOverrideCommandWithArgs(commands); runCommand.disableDetach(); runCommand.setLogDir(container.getLogDir()); } else { @@ -818,6 +834,10 @@ public void launchContainer(ContainerRuntimeContext ctx) runCommand.detachOnRun(); } + if (serviceMode) { + runCommand.setServiceMode(serviceMode); + } + if(enableUserReMapping) { if (!allowPrivilegedContainerExecution(container)) { runCommand.groupAdd(groups); @@ -1279,11 +1299,14 @@ private void handleContainerKill(ContainerRuntimeContext ctx, throw new ContainerExecutionException(e); } + boolean serviceMode = Boolean.parseBoolean(env.get( + ENV_DOCKER_CONTAINER_DOCKER_SERVICE_MODE)); + // Only need to check whether the container was asked to be privileged. // If the container had failed the permissions checks upon launch, it // would have never been launched and thus we wouldn't be here // attempting to signal it. - if (isContainerRequestedAsPrivileged(container)) { + if (isContainerRequestedAsPrivileged(container) || serviceMode) { String containerId = container.getContainerId().toString(); DockerCommandExecutor.DockerContainerStatus containerStatus = DockerCommandExecutor.getContainerStatus(containerId, diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/docker/DockerRunCommand.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/docker/DockerRunCommand.java index b0603a3..7fb0e40 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/docker/DockerRunCommand.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/docker/DockerRunCommand.java @@ -199,6 +199,12 @@ public DockerRunCommand setLogDir(String logDir) { return this; } + public DockerRunCommand setServiceMode(boolean serviceMode) { + String value = Boolean.toString(serviceMode); + super.addCommandArguments("service-mode", value); + return this; + } + /** * Check if user defined environment variables are empty. * diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/container-executor.h b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/container-executor.h index b215af7..757bd16 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/container-executor.h +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/container-executor.h @@ -325,12 +325,6 @@ int sync_yarn_sysfs(char* const* local_dirs, const char *running_user, */ int execute_regex_match(const char *regex_str, const char *input); -/** - * Validate the docker image name matches the expected input. - * Return 0 on success. - */ -int validate_docker_image_name(const char *image_name); - struct configuration* get_cfg(); /** diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/utils/docker-util.c b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/utils/docker-util.c index 1711433..1e52f3f 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/utils/docker-util.c +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/utils/docker-util.c @@ -28,6 +28,7 @@ #include "docker-util.h" #include "string-utils.h" #include "util.h" +#include "container-executor.h" #include #include #include @@ -374,6 +375,8 @@ const char *get_docker_error_message(const int error_code) { return "Invalid docker tmpfs mount"; case INVALID_DOCKER_RUNTIME: return "Invalid docker runtime"; + case SERVICE_MODE_DISABLED: + return "Service mode disabled"; default: return "Unknown error"; } @@ -987,6 +990,22 @@ static int set_runtime(const struct configuration *command_config, return ret; } +int is_service_mode_enabled(const struct configuration *command_config, + const struct configuration *executor_cfg, args *args) { + int ret = 0; + struct section *section = get_configuration_section(CONTAINER_EXECUTOR_CFG_DOCKER_SECTION, command_config); + char *value = get_configuration_value("service-mode", DOCKER_COMMAND_FILE_SECTION, command_config); + if (value != NULL && strcasecmp(value, "true") == 0) { + if (is_feature_enabled(DOCKER_SERVICE_MODE_ENABLED_KEY, ret, section)) { + ret = 1; + } else { + ret = SERVICE_MODE_DISABLED; + } + } + free(value); + return ret; +} + static int add_ports_mapping_to_command(const struct configuration *command_config, args *args) { int i = 0, ret = 0; char *network_type = (char*) malloc(128); @@ -1595,12 +1614,19 @@ int get_docker_run_command(const char *command_file, const struct configuration char *privileged = NULL; char *no_new_privileges_enabled = NULL; char *use_entry_point = NULL; + int service_mode_enabled = 0; struct configuration command_config = {0, NULL}; ret = read_and_verify_command_file(command_file, DOCKER_RUN_COMMAND, &command_config); if (ret != 0) { goto free_and_exit; } + service_mode_enabled = is_service_mode_enabled(&command_config, conf, args); + if (service_mode_enabled == SERVICE_MODE_DISABLED) { + ret = SERVICE_MODE_DISABLED; + goto free_and_exit; + } + use_entry_point = get_configuration_value("use-entry-point", DOCKER_COMMAND_FILE_SECTION, &command_config); if (use_entry_point != NULL && strcasecmp(use_entry_point, "true") == 0) { entry_point = 1; @@ -1612,10 +1638,13 @@ int get_docker_run_command(const char *command_file, const struct configuration ret = INVALID_DOCKER_CONTAINER_NAME; goto free_and_exit; } - user = get_configuration_value("user", DOCKER_COMMAND_FILE_SECTION, &command_config); - if (user == NULL) { - ret = INVALID_DOCKER_USER_NAME; - goto free_and_exit; + + if (!service_mode_enabled) { + user = get_configuration_value("user", DOCKER_COMMAND_FILE_SECTION, &command_config); + if (user == NULL) { + ret = INVALID_DOCKER_USER_NAME; + goto free_and_exit; + } } image = get_configuration_value("image", DOCKER_COMMAND_FILE_SECTION, &command_config); if (image == NULL || validate_docker_image_name(image) != 0) { @@ -1640,12 +1669,14 @@ int get_docker_run_command(const char *command_file, const struct configuration privileged = get_configuration_value("privileged", DOCKER_COMMAND_FILE_SECTION, &command_config); if (privileged == NULL || strcmp(privileged, "false") == 0) { - char *user_buffer = make_string("--user=%s", user); - ret = add_to_args(args, user_buffer); - free(user_buffer); - if (ret != 0) { - ret = BUFFER_TOO_SMALL; - goto free_and_exit; + if (!service_mode_enabled) { + char *user_buffer = make_string("--user=%s", user); + ret = add_to_args(args, user_buffer); + free(user_buffer); + if (ret != 0) { + ret = BUFFER_TOO_SMALL; + goto free_and_exit; + } } no_new_privileges_enabled = get_configuration_value("docker.no-new-privileges.enabled", @@ -1725,9 +1756,11 @@ int get_docker_run_command(const char *command_file, const struct configuration goto free_and_exit; } - ret = set_group_add(&command_config, args); - if (ret != 0) { - goto free_and_exit; + if (!service_mode_enabled) { + ret = set_group_add(&command_config, args); + if (ret != 0) { + goto free_and_exit; + } } ret = set_devices(&command_config, conf, args); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/utils/docker-util.h b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/utils/docker-util.h index 07da195..d9d34a0 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/utils/docker-util.h +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/utils/docker-util.h @@ -36,6 +36,7 @@ #define DOCKER_START_COMMAND "start" #define DOCKER_EXEC_COMMAND "exec" #define DOCKER_IMAGES_COMMAND "images" +#define DOCKER_SERVICE_MODE_ENABLED_KEY "docker.service-mode.enabled" #define DOCKER_ARG_MAX 1024 #define ARGS_INITIAL_VALUE { 0 }; @@ -71,7 +72,8 @@ enum docker_error_codes { INVALID_PID_NAMESPACE, INVALID_DOCKER_IMAGE_TRUST, INVALID_DOCKER_TMPFS_MOUNT, - INVALID_DOCKER_RUNTIME + INVALID_DOCKER_RUNTIME, + SERVICE_MODE_DISABLED }; /** -- 2.7.4 (Apple Git-66)