args) {
+ this.names = names;
+ this.action = action;
+ this.args = args;
+ }
+
+ Syscall() {
+ this(null, null, null);
+ }
+
+ @JsonSerialize(include = JsonSerialize.Inclusion.NON_DEFAULT)
+ static class SeccompArg {
+ final private int index;
+ final private long value;
+ final private long valueTwo;
+ final private String op;
+
+ public int getIndex() {
+ return index;
+ }
+
+ public long getValue() {
+ return value;
+ }
+
+ public long getValueTwo() {
+ return valueTwo;
+ }
+
+ public String getOp() {
+ return op;
+ }
+
+ SeccompArg(int index, long value, long valueTwo, String op) {
+ this.index = index;
+ this.value = value;
+ this.valueTwo = valueTwo;
+ this.op = op;
+ }
+
+ SeccompArg() {
+ this(0, 0, 0, null);
+ }
+ }
+ }
+ }
+ }
+ }
+}
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/OCIContainerRuntime.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/OCIContainerRuntime.java
index 45105f7ad17..530f66e4d01 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/OCIContainerRuntime.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/OCIContainerRuntime.java
@@ -53,6 +53,7 @@
import java.util.regex.Pattern;
import static org.apache.hadoop.yarn.server.nodemanager.containermanager.linux.runtime.DockerLinuxContainerRuntime.isDockerContainerRequested;
+import static org.apache.hadoop.yarn.server.nodemanager.containermanager.linux.runtime.RuncContainerRuntime.isRuncContainerRequested;
/**
* This class is a {@link ContainerRuntime} implementation that uses the
@@ -117,7 +118,8 @@ public void initialize(Configuration conf, Context nmContext)
public static boolean isOCICompliantContainerRequested(
Configuration daemonConf, Map env) {
- return isDockerContainerRequested(daemonConf, env);
+ return isDockerContainerRequested(daemonConf, env) ||
+ isRuncContainerRequested(daemonConf, env);
}
@VisibleForTesting
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/RuncContainerRuntime.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/RuncContainerRuntime.java
new file mode 100644
index 00000000000..0ab5b02abcd
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/RuncContainerRuntime.java
@@ -0,0 +1,846 @@
+/*
+ *
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ * /
+ */
+
+package org.apache.hadoop.yarn.server.nodemanager.containermanager.linux.runtime;
+
+import com.google.common.annotations.VisibleForTesting;
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.classification.InterfaceAudience;
+import org.apache.hadoop.classification.InterfaceStability;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.hdfs.protocol.datatransfer.IOStreamPair;
+import org.apache.hadoop.security.authorize.AccessControlList;
+import org.apache.hadoop.util.StringUtils;
+import org.apache.hadoop.util.concurrent.HadoopExecutors;
+import org.apache.hadoop.yarn.api.records.ApplicationId;
+import org.apache.hadoop.yarn.api.records.ContainerId;
+import org.apache.hadoop.yarn.api.records.LocalResource;
+import org.apache.hadoop.yarn.conf.YarnConfiguration;
+import org.apache.hadoop.yarn.exceptions.YarnException;
+import org.apache.hadoop.yarn.server.nodemanager.ContainerExecutor;
+import org.apache.hadoop.yarn.server.nodemanager.Context;
+import org.apache.hadoop.yarn.server.nodemanager.containermanager.container.Container;
+import org.apache.hadoop.yarn.server.nodemanager.containermanager.launcher.ContainerLaunch;
+import org.apache.hadoop.yarn.server.nodemanager.containermanager.linux.privileged.PrivilegedOperation;
+import org.apache.hadoop.yarn.server.nodemanager.containermanager.linux.privileged.PrivilegedOperationException;
+import org.apache.hadoop.yarn.server.nodemanager.containermanager.linux.privileged.PrivilegedOperationExecutor;
+import org.apache.hadoop.yarn.server.nodemanager.containermanager.linux.resources.CGroupsHandler;
+import org.apache.hadoop.yarn.server.nodemanager.containermanager.linux.resources.ResourceHandlerModule;
+import org.apache.hadoop.yarn.server.nodemanager.containermanager.localizer.LocalResourceRequest;
+import org.apache.hadoop.yarn.server.nodemanager.containermanager.localizer.LocalizedResource;
+import org.apache.hadoop.yarn.server.nodemanager.containermanager.localizer.ResourceLocalizationService;
+import org.apache.hadoop.yarn.server.nodemanager.containermanager.runtime.ContainerExecutionException;
+import org.apache.hadoop.yarn.server.nodemanager.containermanager.runtime.ContainerRuntimeConstants;
+import org.apache.hadoop.yarn.server.nodemanager.containermanager.runtime.ContainerRuntimeContext;
+import org.apache.hadoop.yarn.server.nodemanager.containermanager.linux.runtime.OCIContainerExecutorConfig.OCIRuntimeConfig;
+import org.apache.hadoop.yarn.server.nodemanager.containermanager.linux.runtime.OCIContainerExecutorConfig.OCILayer;
+import org.apache.hadoop.yarn.server.nodemanager.containermanager.linux.runtime.OCIContainerExecutorConfig.OCIRuntimeConfig.OCILinuxConfig;
+import org.apache.hadoop.yarn.server.nodemanager.containermanager.linux.runtime.OCIContainerExecutorConfig.OCIRuntimeConfig.OCIMount;
+import org.apache.hadoop.yarn.server.nodemanager.containermanager.linux.runtime.OCIContainerExecutorConfig.OCIRuntimeConfig.OCIProcessConfig;
+import org.apache.hadoop.yarn.server.nodemanager.containermanager.volume.csi.ContainerVolumePublisher;
+import org.apache.hadoop.yarn.server.nodemanager.executor.ContainerExecContext;
+import org.codehaus.jackson.JsonNode;
+import org.codehaus.jackson.map.ObjectMapper;
+
+import java.io.File;
+import java.io.IOException;
+import java.net.URISyntaxException;
+import java.nio.charset.StandardCharsets;
+import java.nio.file.Files;
+import java.nio.file.Paths;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+import java.util.concurrent.ScheduledExecutorService;
+import java.util.concurrent.TimeUnit;
+
+import static org.apache.hadoop.yarn.conf.YarnConfiguration.DEFAULT_NM_RUNC_IMAGE_TAG_TO_MANIFEST_PLUGIN;
+import static org.apache.hadoop.yarn.conf.YarnConfiguration.DEFAULT_NM_RUNC_LAYER_MOUNTS_TO_KEEP;
+import static org.apache.hadoop.yarn.conf.YarnConfiguration.DEFAULT_NM_REAP_RUNC_LAYER_MOUNTS_INTERVAL;
+import static org.apache.hadoop.yarn.conf.YarnConfiguration.DEFAULT_NM_RUNC_MANIFEST_TO_RESOURCES_PLUGIN;
+import static org.apache.hadoop.yarn.conf.YarnConfiguration.NM_RUNC_IMAGE_TAG_TO_MANIFEST_PLUGIN;
+import static org.apache.hadoop.yarn.conf.YarnConfiguration.NM_RUNC_LAYER_MOUNTS_TO_KEEP;
+import static org.apache.hadoop.yarn.conf.YarnConfiguration.NM_RUNC_MANIFEST_TO_RESOURCES_PLUGIN;
+import static org.apache.hadoop.yarn.conf.YarnConfiguration.NM_REAP_RUNC_LAYER_MOUNTS_INTERVAL;
+import static org.apache.hadoop.yarn.server.nodemanager.containermanager.linux.runtime.LinuxContainerRuntimeConstants.*;
+/**
+ * This class is an extension of {@link OCIContainerRuntime} that uses the
+ * native {@code container-executor} binary via a
+ * {@link PrivilegedOperationExecutor} instance to launch processes inside
+ * Runc containers.
+ *
+ * The following environment variables are used to configure the Runc
+ * engine:
+ *
+ *
+ * -
+ * {@code YARN_CONTAINER_RUNTIME_RUNC_IMAGE} names which image
+ * will be used to launch the Runc container.
+ *
+ * -
+ * {@code YARN_CONTAINER_RUNTIME_RUNC_CONTAINER_PID_NAMESPACE}
+ * controls which PID namespace will be used by the Runc container. By
+ * default, each Runc container has its own PID namespace. To share the
+ * namespace of the host, the
+ * {@code yarn.nodemanager.runtime.linux.runc.host-pid-namespace.allowed}
+ * property must be set to {@code true}. If the host PID namespace is
+ * allowed and this environment variable is set to {@code host}, the
+ * Runc container will share the host's PID namespace. No other value is
+ * allowed.
+ *
+ * -
+ * {@code YARN_CONTAINER_RUNTIME_RUNC_CONTAINER_HOSTNAME} sets the
+ * hostname to be used by the Runc container. If not specified, a
+ * hostname will be derived from the container ID and set as default
+ * hostname for networks other than 'host'.
+ *
+ * -
+ * {@code YARN_CONTAINER_RUNTIME_RUNC_RUN_PRIVILEGED_CONTAINER}
+ * controls whether the Runc container is a privileged container. In order
+ * to use privileged containers, the
+ * {@code yarn.nodemanager.runtime.linux.runc.privileged-containers.allowed}
+ * property must be set to {@code true}, and the application owner must
+ * appear in the value of the
+ * {@code yarn.nodemanager.runtime.linux.runc.privileged-containers.acl}
+ * property. If this environment variable is set to {@code true}, a
+ * privileged Runc container will be used if allowed. No other value is
+ * allowed, so the environment variable should be left unset rather than
+ * setting it to false.
+ *
+ *
+ */
+@InterfaceAudience.Private
+@InterfaceStability.Unstable
+public class RuncContainerRuntime extends OCIContainerRuntime {
+
+ private static final Log LOG = LogFactory.getLog(
+ RuncContainerRuntime.class);
+
+ @InterfaceAudience.Private
+ private static final String RUNTIME_TYPE = "RUNC";
+
+ @InterfaceAudience.Private
+ public static final String ENV_RUNC_CONTAINER_IMAGE =
+ "YARN_CONTAINER_RUNTIME_RUNC_IMAGE";
+ @InterfaceAudience.Private
+ public static final String ENV_RUNC_CONTAINER_HOSTNAME =
+ "YARN_CONTAINER_RUNTIME_RUNC_CONTAINER_HOSTNAME";
+
+ @InterfaceAudience.Private
+ public final static String ENV_RUNC_CONTAINER_PID_NAMESPACE =
+ formatOciEnvKey(RUNTIME_TYPE, CONTAINER_PID_NAMESPACE_SUFFIX);
+ @InterfaceAudience.Private
+ public final static String ENV_RUNC_CONTAINER_RUN_PRIVILEGED_CONTAINER =
+ formatOciEnvKey(RUNTIME_TYPE, RUN_PRIVILEGED_CONTAINER_SUFFIX);
+
+ private Configuration conf;
+ private Context nmContext;
+ private PrivilegedOperationExecutor privilegedOperationExecutor;
+ private CGroupsHandler cGroupsHandler;
+ private RuncImageTagToManifestPlugin imageTagToManifestPlugin;
+ private RuncManifestToResourcesPlugin manifestToResourcesPlugin;
+ private ObjectMapper mapper;
+ private String seccomp;
+ private int layersToKeep;
+ private String defaultRuncImage;
+ private ScheduledExecutorService exec;
+ private String seccompProfile;
+ private Set defaultROMounts = new HashSet<>();
+ private Set defaultRWMounts = new HashSet<>();
+ private Set allowedNetworks = new HashSet<>();
+ private Set allowedRuntimes = new HashSet<>();
+ private AccessControlList privilegedContainersAcl;
+
+ public RuncContainerRuntime(PrivilegedOperationExecutor
+ privilegedOperationExecutor) {
+ this(privilegedOperationExecutor, ResourceHandlerModule
+ .getCGroupsHandler());
+ }
+
+ //A constructor with an injected cGroupsHandler primarily used for testing.
+ @VisibleForTesting
+ public RuncContainerRuntime(PrivilegedOperationExecutor
+ privilegedOperationExecutor, CGroupsHandler cGroupsHandler) {
+ super(privilegedOperationExecutor, cGroupsHandler);
+ this.privilegedOperationExecutor = privilegedOperationExecutor;
+
+ if (cGroupsHandler == null) {
+ LOG.info("cGroupsHandler is null - cgroups not in use.");
+ } else {
+ this.cGroupsHandler = cGroupsHandler;
+ }
+ }
+
+ @Override
+ public void initialize(Configuration conf, Context nmContext)
+ throws ContainerExecutionException {
+ super.initialize(conf, nmContext);
+ this.conf = conf;
+ this.nmContext = nmContext;
+ imageTagToManifestPlugin = chooseImageTagToManifestPlugin();
+ imageTagToManifestPlugin.init(conf);
+ manifestToResourcesPlugin = chooseManifestToResourcesPlugin();
+ manifestToResourcesPlugin.init(conf);
+ mapper = new ObjectMapper();
+ defaultRuncImage = conf.get(YarnConfiguration.NM_RUNC_IMAGE_NAME);
+
+ allowedNetworks.clear();
+ allowedRuntimes.clear();
+
+ allowedNetworks.addAll(Arrays.asList(
+ conf.getTrimmedStrings(
+ YarnConfiguration.NM_RUNC_ALLOWED_CONTAINER_NETWORKS,
+ YarnConfiguration.DEFAULT_NM_RUNC_ALLOWED_CONTAINER_NETWORKS)));
+
+ allowedRuntimes.addAll(Arrays.asList(
+ conf.getTrimmedStrings(
+ YarnConfiguration.NM_RUNC_ALLOWED_CONTAINER_RUNTIMES,
+ YarnConfiguration.DEFAULT_NM_RUNC_ALLOWED_CONTAINER_RUNTIMES)));
+
+ privilegedContainersAcl = new AccessControlList(conf.getTrimmed(
+ YarnConfiguration.NM_RUNC_PRIVILEGED_CONTAINERS_ACL,
+ YarnConfiguration.DEFAULT_NM_RUNC_PRIVILEGED_CONTAINERS_ACL));
+
+ seccompProfile = conf.get(YarnConfiguration.NM_RUNC_SECCOMP_PROFILE);
+
+ defaultROMounts.addAll(Arrays.asList(
+ conf.getTrimmedStrings(
+ YarnConfiguration.NM_RUNC_DEFAULT_RO_MOUNTS)));
+
+ defaultRWMounts.addAll(Arrays.asList(
+ conf.getTrimmedStrings(
+ YarnConfiguration.NM_RUNC_DEFAULT_RW_MOUNTS)));
+
+ try {
+ //TODO Remove whitespace in seccomp that gets output to config.json
+ if (seccompProfile != null) {
+ seccomp = new String(Files.readAllBytes(Paths.get(seccompProfile)),
+ StandardCharsets.UTF_8);
+ }
+ } catch (IOException ioe) {
+ throw new ContainerExecutionException(ioe);
+ }
+
+ layersToKeep = conf.getInt(NM_RUNC_LAYER_MOUNTS_TO_KEEP,
+ DEFAULT_NM_RUNC_LAYER_MOUNTS_TO_KEEP);
+
+ }
+
+ @Override
+ public void start() {
+ int reapRuncLayerMountsInterval =
+ conf.getInt(NM_REAP_RUNC_LAYER_MOUNTS_INTERVAL,
+ DEFAULT_NM_REAP_RUNC_LAYER_MOUNTS_INTERVAL);
+ exec = HadoopExecutors.newScheduledThreadPool(1);
+ exec.scheduleAtFixedRate(
+ new Runnable() {
+ @Override
+ public void run() {
+ try {
+ PrivilegedOperation launchOp = new PrivilegedOperation(
+ PrivilegedOperation.OperationType.REAP_RUNC_LAYER_MOUNTS);
+ launchOp.appendArgs(Integer.toString(layersToKeep));
+ try {
+ String stdout = privilegedOperationExecutor
+ .executePrivilegedOperation(null,
+ launchOp, null, null, false, false);
+ if(stdout != null) {
+ LOG.info("Reap layer mounts thread: " + stdout);
+ }
+ } catch (PrivilegedOperationException e) {
+ LOG.warn("Failed to reap old runc layer mounts", e);
+ }
+ } catch (Exception e) {
+ LOG.warn("Reap layer mount thread caught an exception: ", e);
+ }
+ }
+ }, 0, reapRuncLayerMountsInterval, TimeUnit.SECONDS);
+ imageTagToManifestPlugin.start();
+ manifestToResourcesPlugin.start();
+ }
+
+ @Override
+ public void stop() {
+ exec.shutdownNow();
+ imageTagToManifestPlugin.stop();
+ manifestToResourcesPlugin.stop();
+ }
+
+ @Override
+ public void launchContainer(ContainerRuntimeContext ctx)
+ throws ContainerExecutionException {
+ List env = new ArrayList<>();
+ Container container = ctx.getContainer();
+ String user = container.getUser();
+ ContainerId containerId = container.getContainerId();
+ ApplicationId appId = containerId.getApplicationAttemptId()
+ .getApplicationId();
+
+ Map environment = container.getLaunchContext()
+ .getEnvironment();
+ ArrayList mounts = new ArrayList<>();
+ ArrayList layers = new ArrayList<>();
+ String hostname = environment.get(ENV_RUNC_CONTAINER_HOSTNAME);
+
+ validateHostname(hostname);
+
+ String containerIdStr = containerId.toString();
+ String applicationId = appId.toString();
+ Path containerWorkDir = ctx.getExecutionAttribute(CONTAINER_WORK_DIR);
+
+ RuncRuntimeObject runcRuntimeObject =
+ container.getContainerRuntimeData(RuncRuntimeObject.class);
+ List layerResources = runcRuntimeObject.getOCILayers();
+
+ ResourceLocalizationService localizationService =
+ nmContext.getContainerManager().getResourceLocalizationService();
+
+ List args = new ArrayList<>();
+
+ try {
+ try {
+ LocalResource rsrc = runcRuntimeObject.getConfig();
+ LocalResourceRequest req = new LocalResourceRequest(rsrc);
+ LocalizedResource localRsrc = localizationService
+ .getLocalizedResource(req, user, appId);
+ if (localRsrc == null) {
+ throw new ContainerExecutionException("Could not successfully " +
+ "localize layers. rsrc: " + rsrc.getResource().getFile());
+ }
+
+ File file = new File(localRsrc.getLocalPath().toString());
+ List imageEnv = extractImageEnv(file);
+ if (imageEnv != null && !imageEnv.isEmpty()) {
+ env.addAll(imageEnv);
+ }
+ List entrypoint = extractImageEntrypoint(file);
+ if (entrypoint != null && !entrypoint.isEmpty()) {
+ args.addAll(entrypoint);
+ }
+ } catch (IOException ioe) {
+ throw new ContainerExecutionException(ioe);
+ }
+
+ for (LocalResource rsrc : layerResources) {
+ LocalResourceRequest req = new LocalResourceRequest(rsrc);
+ LocalizedResource localRsrc = localizationService
+ .getLocalizedResource(req, user, appId);
+
+ OCILayer layer = new OCILayer("application/vnd.squashfs",
+ localRsrc.getLocalPath().toString());
+ layers.add(layer);
+ }
+ } catch (URISyntaxException e) {
+ throw new ContainerExecutionException(e);
+ }
+
+ setContainerMounts(mounts, ctx, containerWorkDir);
+
+ String resourcesOpts = ctx.getExecutionAttribute(RESOURCES_OPTIONS);
+
+ Path nmPrivateContainerScriptPath = ctx.getExecutionAttribute(
+ NM_PRIVATE_CONTAINER_SCRIPT_PATH);
+
+ Path nmPrivateTokensPath =
+ ctx.getExecutionAttribute(NM_PRIVATE_TOKENS_PATH);
+
+ int cpuShares = container.getResource().getVirtualCores();
+
+ // Zero sets to default of 1024. 2 is the minimum value otherwise
+ if (cpuShares < 2) {
+ cpuShares = 2;
+ }
+
+ Path launchDst =
+ new Path(containerWorkDir, ContainerLaunch.CONTAINER_SCRIPT);
+
+ args.add("bash");
+ args.add(launchDst.toUri().getPath());
+
+ String cgroupPath = getCgroupPath(resourcesOpts, "runc-" + containerIdStr);
+
+ String pidFile = ctx.getExecutionAttribute(PID_FILE_PATH).toString();
+
+ @SuppressWarnings("unchecked")
+ List localDirs = ctx.getExecutionAttribute(LOCAL_DIRS);
+ @SuppressWarnings("unchecked")
+ List logDirs = ctx.getExecutionAttribute(LOG_DIRS);
+
+ Path keystorePath = ctx.getExecutionAttribute(NM_PRIVATE_KEYSTORE_PATH);
+ Path truststorePath = ctx.getExecutionAttribute(NM_PRIVATE_TRUSTSTORE_PATH);
+
+ int https = 0;
+ String keystore = null;
+ String truststore = null;
+
+ if (keystorePath != null && truststorePath != null) {
+ https = 1;
+ keystore = keystorePath.toUri().getPath();
+ truststore = truststorePath.toUri().getPath();
+ }
+
+ OCIProcessConfig processConfig = createOCIProcessConfig(
+ containerWorkDir.toString(), env, args);
+ OCILinuxConfig linuxConfig = createOCILinuxConfig(cpuShares,
+ cgroupPath, seccomp);
+
+ OCIRuntimeConfig ociRuntimeConfig = new OCIRuntimeConfig(null, mounts,
+ processConfig, hostname, null, null, linuxConfig);
+
+ OCIContainerExecutorConfig ociContainerExecutorConfig =
+ createOCIContainerExecutorConfig(user, containerIdStr, applicationId,
+ pidFile, nmPrivateContainerScriptPath.toString(),
+ nmPrivateTokensPath.toString(), https, keystore, truststore,
+ localDirs, logDirs, layers,
+ ociRuntimeConfig);
+
+ String commandFile = writeCommandToFile(mapper,
+ ociContainerExecutorConfig, container, nmContext);
+ PrivilegedOperation launchOp = new PrivilegedOperation(
+ PrivilegedOperation.OperationType.RUN_RUNC_CONTAINER);
+
+ launchOp.appendArgs(commandFile);
+
+ try {
+ privilegedOperationExecutor.executePrivilegedOperation(null,
+ launchOp, null, null, false, false);
+ } catch (PrivilegedOperationException e) {
+ LOG.info("Launch container failed: ", e);
+ try {
+ LOG.debug("config.json used: " +
+ mapper.writeValueAsString(ociContainerExecutorConfig));
+ } catch (IOException ioe) {
+ LOG.info("Json Generation Exception", ioe);
+ }
+
+ throw new ContainerExecutionException("Launch container failed", e
+ .getExitCode(), e.getOutput(), e.getErrorOutput());
+ }
+ }
+
+ private String getCgroupPath(String resourcesOptions, String containerIdStr) {
+ if (cGroupsHandler == null) {
+ LOG.debug("cGroupsHandler is null. cgroups are not in use. nothing to"
+ + " do.");
+ return null;
+ }
+
+ if (resourcesOptions.equals(
+ (PrivilegedOperation.CGROUP_ARG_PREFIX + PrivilegedOperation
+ .CGROUP_ARG_NO_TASKS))) {
+ LOG.debug("no resource restrictions specified. not using runc's "
+ + "cgroup options");
+ } else {
+ LOG.debug("using runc's cgroups options");
+
+ String cGroupPath = "/" + cGroupsHandler.getRelativePathForCGroup(
+ containerIdStr);
+
+ LOG.debug("using cgroup parent: " + cGroupPath);
+
+ return cGroupPath;
+ }
+ return null;
+ }
+
+ private void addDefaultMountLocation(List mounts,
+ Set defaultMounts, boolean createSource, boolean isReadWrite)
+ throws ContainerExecutionException {
+ if(defaultMounts != null && !defaultMounts.isEmpty()) {
+ for (String mount : defaultMounts) {
+ String[] dir = StringUtils.split(mount, ':');
+ if (dir.length != 2) {
+ throw new ContainerExecutionException("Invalid mount : " +
+ mount);
+ }
+ String src = dir[0];
+ String dst = dir[1];
+ addRuncMountLocation(mounts, src, dst, createSource, isReadWrite);
+ }
+ }
+ }
+
+ private void addRuncMountLocation(List mounts, String srcPath,
+ String dstPath, boolean createSource, boolean isReadWrite) {
+ if (!createSource) {
+ boolean sourceExists = new File(srcPath).exists();
+ if (!sourceExists) {
+ return;
+ }
+ }
+
+ ArrayList options = new ArrayList<>();
+ if (isReadWrite) {
+ options.add("rw");
+ } else {
+ options.add("ro");
+ }
+ options.add("rbind");
+ options.add("rprivate");
+ mounts.add(new OCIMount(dstPath, "bind", srcPath, options));
+ }
+
+ private void addAllRuncMountLocations(List mounts,
+ List paths, boolean createSource, boolean isReadWrite) {
+ for (String dir: paths) {
+ this.addRuncMountLocation(mounts, dir, dir, createSource, isReadWrite);
+ }
+ }
+
+ public Map getLocalResources(
+ Container container) throws IOException {
+ Map containerLocalRsrc =
+ container.getLaunchContext().getLocalResources();
+ long layerCount = 0;
+ Map environment =
+ container.getLaunchContext().getEnvironment();
+ String imageName = environment.get(ENV_RUNC_CONTAINER_IMAGE);
+ if (imageName == null || imageName.isEmpty()) {
+ environment.put(ENV_RUNC_CONTAINER_IMAGE,
+ defaultRuncImage);
+ imageName = defaultRuncImage;
+ }
+
+ ImageManifest manifest =
+ imageTagToManifestPlugin.getManifestFromImageTag(imageName);
+ LocalResource config =
+ manifestToResourcesPlugin.getConfigResource(manifest);
+ List layers =
+ manifestToResourcesPlugin.getLayerResources(manifest);
+
+ RuncRuntimeObject runcRuntimeObject =
+ new RuncRuntimeObject(config, layers);
+ container.setContainerRuntimeData(runcRuntimeObject);
+
+ for (LocalResource localRsrc : layers) {
+ while(containerLocalRsrc.putIfAbsent("runc-layer" +
+ Long.toString(layerCount), localRsrc) != null) {
+ layerCount++;
+ }
+ }
+
+ while(containerLocalRsrc.putIfAbsent("runc-config" +
+ Long.toString(layerCount), config) != null) {
+ layerCount++;
+ }
+
+ return containerLocalRsrc;
+ }
+
+ protected RuncImageTagToManifestPlugin chooseImageTagToManifestPlugin()
+ throws ContainerExecutionException {
+ String pluginName =
+ conf.get(NM_RUNC_IMAGE_TAG_TO_MANIFEST_PLUGIN,
+ DEFAULT_NM_RUNC_IMAGE_TAG_TO_MANIFEST_PLUGIN);
+ RuncImageTagToManifestPlugin runcImageTagToManifestPlugin;
+ try {
+ Class> clazz = Class.forName(pluginName);
+ runcImageTagToManifestPlugin =
+ (RuncImageTagToManifestPlugin) clazz.newInstance();
+ } catch (Exception e) {
+ throw new ContainerExecutionException(e);
+ }
+ return runcImageTagToManifestPlugin;
+ }
+
+ protected RuncManifestToResourcesPlugin chooseManifestToResourcesPlugin()
+ throws ContainerExecutionException {
+ String pluginName =
+ conf.get(NM_RUNC_MANIFEST_TO_RESOURCES_PLUGIN,
+ DEFAULT_NM_RUNC_MANIFEST_TO_RESOURCES_PLUGIN);
+ LOG.info("pluginName = " + pluginName);
+ RuncManifestToResourcesPlugin runcManifestToResourcesPlugin;
+ try {
+ Class> clazz = Class.forName(pluginName);
+ runcManifestToResourcesPlugin =
+ (RuncManifestToResourcesPlugin) clazz.newInstance();
+ } catch (Exception e) {
+ throw new ContainerExecutionException(e);
+ }
+ return runcManifestToResourcesPlugin;
+ }
+
+ @SuppressWarnings("unchecked")
+ protected List extractImageEnv(File config) throws IOException {
+ JsonNode node = mapper.readTree(config);
+ JsonNode envNode = node.path("config").path("Env");
+ if (envNode.isMissingNode()) {
+ return null;
+ }
+ return mapper.readValue(envNode, List.class);
+ }
+
+ @SuppressWarnings("unchecked")
+ protected List extractImageEntrypoint(File config)
+ throws IOException {
+ JsonNode node = mapper.readTree(config);
+ JsonNode entrypointNode = node.path("config").path("Entrypoint");
+ if (entrypointNode.isMissingNode()) {
+ return null;
+ }
+ return mapper.readValue(entrypointNode, List.class);
+ }
+
+ private OCIContainerExecutorConfig createOCIContainerExecutorConfig(
+ String username, String containerId, String applicationId, String pidFile,
+ String containerScriptPath, String containerCredentialsPath,
+ int https, String keystorePath, String truststorePath,
+ List localDirs, List logDirs,
+ List layers, OCIRuntimeConfig ociRuntimeConfig) {
+
+ return new OCIContainerExecutorConfig(username, containerId, applicationId,
+ pidFile, containerScriptPath, containerCredentialsPath,
+ https, keystorePath, truststorePath,
+ localDirs, logDirs, layers, layersToKeep, ociRuntimeConfig);
+ }
+
+ private OCIProcessConfig createOCIProcessConfig(String cwd,
+ List env, List args) {
+ return new OCIProcessConfig(false, null, cwd, env,
+ args, null, null, null, false, 0, null, null);
+ }
+
+ private OCILinuxConfig createOCILinuxConfig(long cpuShares,
+ String cgroupsPath, String seccomp) {
+ OCILinuxConfig.Resources.CPU cgroupCPU =
+ new OCILinuxConfig.Resources.CPU(cpuShares, 0, 0, 0, 0,
+ null, null);
+ OCILinuxConfig.Resources cgroupResources =
+ new OCILinuxConfig.Resources(null, null, cgroupCPU, null, null, null,
+ null, null);
+
+ return new OCILinuxConfig(null, null, null, null,
+ cgroupsPath, cgroupResources, null, null, seccomp, null, null,
+ null, null);
+ }
+
+ private void setContainerMounts(ArrayList mounts,
+ ContainerRuntimeContext ctx, Path containerWorkDir)
+ throws ContainerExecutionException {
+ @SuppressWarnings("unchecked")
+ List filecacheDirs = ctx.getExecutionAttribute(FILECACHE_DIRS);
+ @SuppressWarnings("unchecked")
+ List containerLogDirs = ctx.getExecutionAttribute(
+ CONTAINER_LOG_DIRS);
+ @SuppressWarnings("unchecked")
+ List userFilecacheDirs =
+ ctx.getExecutionAttribute(USER_FILECACHE_DIRS);
+ @SuppressWarnings("unchecked")
+ List applicationLocalDirs =
+ ctx.getExecutionAttribute(APPLICATION_LOCAL_DIRS);
+
+ addRuncMountLocation(mounts, containerWorkDir.toString() +
+ "/private_slash_tmp", "/tmp", true, true);
+ addRuncMountLocation(mounts, containerWorkDir.toString() +
+ "/private_var_slash_tmp", "/var/tmp", true, true);
+
+ addAllRuncMountLocations(mounts, containerLogDirs, true, true);
+ addAllRuncMountLocations(mounts, applicationLocalDirs, true, true);
+ addAllRuncMountLocations(mounts, filecacheDirs, false, false);
+ addAllRuncMountLocations(mounts, userFilecacheDirs, false, false);
+ addDefaultMountLocation(mounts, defaultROMounts, false, false);
+ addDefaultMountLocation(mounts, defaultRWMounts, false, true);
+ }
+
+ public String writeCommandToFile(ObjectMapper mapper,
+ OCIContainerExecutorConfig ociContainerExecutorConfig,
+ Container container, Context nmContext)
+ throws ContainerExecutionException {
+ ContainerId containerId = container.getContainerId();
+ String filePrefix = containerId.toString();
+ ApplicationId appId = containerId.getApplicationAttemptId()
+ .getApplicationId();
+ File commandFile;
+ try {
+ File cmdDir = null;
+
+ if(nmContext != null && nmContext.getLocalDirsHandler() != null) {
+ String cmdDirStr = nmContext.getLocalDirsHandler().getLocalPathForWrite(
+ ResourceLocalizationService.NM_PRIVATE_DIR + Path.SEPARATOR +
+ appId + Path.SEPARATOR + filePrefix + Path.SEPARATOR).toString();
+ cmdDir = new File(cmdDirStr);
+ if (!cmdDir.mkdirs() && !cmdDir.exists()) {
+ throw new IOException("Cannot create container private directory "
+ + cmdDir);
+ }
+ }
+ commandFile = new File(cmdDir + "/runc-config.json");
+
+ try {
+ mapper.writeValue(commandFile, ociContainerExecutorConfig);
+ } catch (IOException ioe) {
+ throw new ContainerExecutionException(ioe);
+ }
+
+ return commandFile.getAbsolutePath();
+ } catch (IOException e) {
+ LOG.warn("Unable to write runc config.json to temporary file!");
+ throw new ContainerExecutionException(e);
+ }
+ }
+
+ public String getExposedPorts(Container container) {
+ return null;
+ }
+
+ public String[] getIpAndHost(Container container) {
+ return null;
+ }
+
+ public IOStreamPair execContainer(ContainerExecContext ctx)
+ throws ContainerExecutionException {
+ return null;
+ }
+
+ public void reapContainer(ContainerRuntimeContext ctx)
+ throws ContainerExecutionException {
+ }
+
+ public void relaunchContainer(ContainerRuntimeContext ctx)
+ throws ContainerExecutionException {
+ }
+
+ /**
+ * Return whether the given environment variables indicate that the operation
+ * is requesting a Runc container. If the environment contains a key
+ * called {@code YARN_CONTAINER_RUNTIME_TYPE} whose value is {@code runc},
+ * this method will return true. Otherwise it will return false.
+ *
+ * @param daemonConf the NodeManager daemon configuration
+ * @param env the environment variable settings for the operation
+ * @return whether a Runc container is requested
+ */
+ public static boolean isRuncContainerRequested(Configuration daemonConf,
+ Map env) {
+ String type = (env == null)
+ ? null : env.get(ContainerRuntimeConstants.ENV_CONTAINER_TYPE);
+ if (type == null) {
+ type = daemonConf.get(YarnConfiguration.LINUX_CONTAINER_RUNTIME_TYPE);
+ }
+ return type != null && type.equals(
+ ContainerRuntimeConstants.CONTAINER_RUNTIME_RUNC);
+ }
+
+
+ @Override
+ public boolean isRuntimeRequested(Map env) {
+ return isRuncContainerRequested(conf, env);
+ }
+
+ @Override
+ public void signalContainer(ContainerRuntimeContext ctx)
+ throws ContainerExecutionException {
+ ContainerExecutor.Signal signal = ctx.getExecutionAttribute(SIGNAL);
+ Container container = ctx.getContainer();
+
+ if (signal == ContainerExecutor.Signal.KILL ||
+ signal == ContainerExecutor.Signal.TERM) {
+ ContainerVolumePublisher publisher = new ContainerVolumePublisher(
+ container, container.getCsiVolumesRootDir(), this);
+ try {
+ publisher.unpublishVolumes();
+ } catch (YarnException | IOException e) {
+ throw new ContainerExecutionException(e);
+ }
+ }
+
+ PrivilegedOperation signalOp = new PrivilegedOperation(
+ PrivilegedOperation.OperationType.SIGNAL_CONTAINER);
+
+ signalOp.appendArgs(ctx.getExecutionAttribute(RUN_AS_USER),
+ ctx.getExecutionAttribute(USER),
+ Integer.toString(PrivilegedOperation.RunAsUserCommand
+ .SIGNAL_CONTAINER.getValue()),
+ ctx.getExecutionAttribute(PID),
+ Integer.toString(signal.getValue()));
+
+ //Some failures here are acceptable. Let the calling executor decide.
+ signalOp.disableFailureLogging();
+
+ try {
+ PrivilegedOperationExecutor executor = PrivilegedOperationExecutor
+ .getInstance(conf);
+
+ executor.executePrivilegedOperation(null,
+ signalOp, null, null, false, false);
+ } catch (PrivilegedOperationException e) {
+ //Don't log the failure here. Some kinds of signaling failures are
+ // acceptable. Let the calling executor decide what to do.
+ throw new ContainerExecutionException("Signal container failed", e
+ .getExitCode(), e.getOutput(), e.getErrorOutput());
+ }
+ }
+
+ static class RuncRuntimeObject {
+ private final List layers;
+ private final LocalResource config;
+
+ RuncRuntimeObject(LocalResource config,
+ List layers) {
+ this.config = config;
+ this.layers = layers;
+ }
+
+ public LocalResource getConfig() {
+ return this.config;
+ }
+
+ public List getOCILayers() {
+ return this.layers;
+ }
+ }
+
+ boolean getHostPidNamespaceEnabled() {
+ return conf.getBoolean(
+ YarnConfiguration.NM_RUNC_ALLOW_HOST_PID_NAMESPACE,
+ YarnConfiguration.DEFAULT_NM_RUNC_ALLOW_HOST_PID_NAMESPACE);
+ }
+
+ boolean getPrivilegedContainersEnabledOnCluster() {
+ return conf.getBoolean(
+ YarnConfiguration.NM_RUNC_ALLOW_PRIVILEGED_CONTAINERS,
+ YarnConfiguration.DEFAULT_NM_RUNC_ALLOW_PRIVILEGED_CONTAINERS);
+ }
+
+ Set getAllowedNetworks() {
+ return allowedNetworks;
+ }
+
+ Set getAllowedRuntimes() {
+ return allowedRuntimes;
+ }
+
+ AccessControlList getPrivilegedContainersAcl() {
+ return privilegedContainersAcl;
+ }
+
+ String getEnvOciContainerPidNamespace() {
+ return ENV_RUNC_CONTAINER_PID_NAMESPACE;
+ }
+
+ String getEnvOciContainerRunPrivilegedContainer() {
+ return ENV_RUNC_CONTAINER_RUN_PRIVILEGED_CONTAINER;
+ }
+}
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/RuncImageTagToManifestPlugin.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/RuncImageTagToManifestPlugin.java
new file mode 100644
index 00000000000..e6cea36ca32
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/RuncImageTagToManifestPlugin.java
@@ -0,0 +1,36 @@
+/*
+ *
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.server.nodemanager.containermanager.linux.runtime;
+
+import org.apache.hadoop.classification.InterfaceStability;
+import org.apache.hadoop.service.Service;
+
+import java.io.IOException;
+
+/**
+ * This class is a plugin interface for the {@link RuncContainerRuntime}
+ * to convert image tags into OCI Image Manifests.
+ */
+@InterfaceStability.Unstable
+public interface RuncImageTagToManifestPlugin extends Service {
+ ImageManifest getManifestFromImageTag(String imageTag) throws IOException;
+
+ String getHashFromImageTag(String imageTag);
+}
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/RuncManifestToResourcesPlugin.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/RuncManifestToResourcesPlugin.java
new file mode 100644
index 00000000000..8e0e2a88b77
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/RuncManifestToResourcesPlugin.java
@@ -0,0 +1,41 @@
+/*
+ *
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.server.nodemanager.containermanager.linux.runtime;
+
+import org.apache.hadoop.classification.InterfaceStability;
+import org.apache.hadoop.service.Service;
+import org.apache.hadoop.yarn.api.records.LocalResource;
+
+import java.io.IOException;
+import java.util.List;
+
+/**
+ * This class is a plugin interface for the {@link RuncContainerRuntime}
+ * that maps OCI Image Manifests into associated config and layers.
+ */
+@InterfaceStability.Unstable
+public interface RuncManifestToResourcesPlugin extends Service {
+ //The layers should be returned in the order in which they
+ // appear in the manifest
+ List getLayerResources(ImageManifest manifest)
+ throws IOException;
+
+ LocalResource getConfigResource(ImageManifest manifest) throws IOException;
+}
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/ResourceLocalizationService.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/ResourceLocalizationService.java
index 3e4af2c7036..36cb452ed6c 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/ResourceLocalizationService.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/ResourceLocalizationService.java
@@ -1699,4 +1699,14 @@ private boolean checkLocalDir(String localDir) {
localDirPathFsPermissionsMap.put(sysDir, nmPrivatePermission);
return localDirPathFsPermissionsMap;
}
+
+ public LocalizedResource getLocalizedResource(LocalResourceRequest req,
+ String user, ApplicationId appId) {
+ LocalResourcesTracker tracker =
+ getLocalResourcesTracker(req.getVisibility(), user, appId);
+ if (tracker == null) {
+ return null;
+ }
+ return tracker.getLocalizedResource(req);
+ }
}
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/runtime/ContainerRuntimeConstants.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/runtime/ContainerRuntimeConstants.java
index a8b295ddb32..9ec92781897 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/runtime/ContainerRuntimeConstants.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/runtime/ContainerRuntimeConstants.java
@@ -34,4 +34,8 @@
@Private
public static final String CONTAINER_RUNTIME_DOCKER =
"docker";
+
+ @Private
+ public static final String CONTAINER_RUNTIME_RUNC =
+ "runc";
}
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/volume/csi/ContainerVolumePublisher.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/volume/csi/ContainerVolumePublisher.java
index 3fec9596e68..283f6a60ab6 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/volume/csi/ContainerVolumePublisher.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/volume/csi/ContainerVolumePublisher.java
@@ -27,7 +27,7 @@
import org.apache.hadoop.yarn.api.records.ResourceInformation;
import org.apache.hadoop.yarn.exceptions.YarnException;
import org.apache.hadoop.yarn.server.nodemanager.containermanager.container.Container;
-import org.apache.hadoop.yarn.server.nodemanager.containermanager.linux.runtime.DockerLinuxContainerRuntime;
+import org.apache.hadoop.yarn.server.nodemanager.containermanager.linux.runtime.OCIContainerRuntime;
import org.apache.hadoop.yarn.server.volume.csi.CsiConstants;
import org.apache.hadoop.yarn.server.volume.csi.VolumeMetaData;
import org.apache.hadoop.yarn.server.volume.csi.exception.InvalidVolumeException;
@@ -51,10 +51,10 @@
private final Container container;
private final String localMountRoot;
- private final DockerLinuxContainerRuntime runtime;
+ private final OCIContainerRuntime runtime;
public ContainerVolumePublisher(Container container, String localMountRoot,
- DockerLinuxContainerRuntime runtime) {
+ OCIContainerRuntime runtime) {
LOG.info("Initiate container volume publisher, containerID={},"
+ " volume local mount rootDir={}",
container.getContainerId().toString(), localMountRoot);
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestLinuxContainerExecutor.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestLinuxContainerExecutor.java
index ae1f574c07f..ae1d0144814 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestLinuxContainerExecutor.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestLinuxContainerExecutor.java
@@ -717,6 +717,14 @@ public void testUpdateYarnSysFS() throws Exception {
verify(lce, times(1)).updateYarnSysFS(ctx, user, appId, spec);
}
+ @Test
+ public void testGetLocalResources() throws Exception {
+ Container container = mock(Container.class);
+ LinuxContainerExecutor lce = mock(LinuxContainerExecutor.class);
+ lce.getLocalResources(container);
+ verify(lce, times(1)).getLocalResources(container);
+ }
+
private static class TestResourceHandler implements LCEResourcesHandler {
static Set postExecContainers = new HashSet();
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/BaseContainerManagerTest.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/BaseContainerManagerTest.java
index 15c1cac9cb8..09c836e596f 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/BaseContainerManagerTest.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/BaseContainerManagerTest.java
@@ -223,6 +223,7 @@ public void setup() throws IOException {
nodeHealthChecker.init(conf);
containerManager = createContainerManager(delSrvc);
((NMContext)context).setContainerManager(containerManager);
+ ((NMContext)context).setContainerExecutor(exec);
nodeStatusUpdater.init(conf);
containerManager.init(conf);
nodeStatusUpdater.start();
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/TestContainerManagerRecovery.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/TestContainerManagerRecovery.java
index e920105abf9..de20abf6682 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/TestContainerManagerRecovery.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/TestContainerManagerRecovery.java
@@ -794,6 +794,7 @@ public int getHttpPort() {
.byteValue() }));
context.getContainerTokenSecretManager().setMasterKey(masterKey);
context.getNMTokenSecretManager().setMasterKey(masterKey);
+ context.setContainerExecutor(exec);
return context;
}
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/container/TestContainer.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/container/TestContainer.java
index ea3acca35e1..a2ef9d9186a 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/container/TestContainer.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/container/TestContainer.java
@@ -1316,6 +1316,7 @@ public boolean matches(LocalizationEvent e) {
new HashMap<>();
private final Map eventToFinalState =
new HashMap<>();
+ private final ContainerExecutor exec;
WrappedContainer(int appId, long timestamp, int id, String user)
throws IOException {
@@ -1352,6 +1353,7 @@ protected void scheduleContainer(Container container) {
container.sendLaunchEvent();
}
};
+ exec = mock(ContainerExecutor.class);
dispatcher.register(LocalizationEventType.class, localizerBus);
dispatcher.register(ContainersLauncherEventType.class, launcherBus);
dispatcher.register(ContainersMonitorEventType.class, monitorBus);
@@ -1412,6 +1414,9 @@ protected void scheduleContainer(Container container) {
}
when(ctxt.getLocalResources()).thenReturn(localResources);
+ when(exec.getLocalResources(any())).thenReturn(localResources);
+ when(context.getContainerExecutor()).thenReturn(exec);
+
if (withServiceData) {
Random r = new Random();
long seed = r.nextLong();
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/TestDockerContainerRuntime.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/TestDockerContainerRuntime.java
index 8ab9df6321b..6e919b68025 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/TestDockerContainerRuntime.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/TestDockerContainerRuntime.java
@@ -30,6 +30,7 @@
import org.apache.hadoop.util.Shell;
import org.apache.hadoop.util.StringUtils;
import org.apache.hadoop.yarn.api.records.ApplicationAttemptId;
+import org.apache.hadoop.yarn.api.records.ApplicationId;
import org.apache.hadoop.yarn.api.records.ContainerId;
import org.apache.hadoop.yarn.api.records.ContainerLaunchContext;
import org.apache.hadoop.yarn.server.nodemanager.LocalDirsHandlerService;
@@ -38,6 +39,7 @@
import org.apache.hadoop.yarn.security.TestDockerClientConfigHandler;
import org.apache.hadoop.yarn.server.nodemanager.ContainerExecutor;
import org.apache.hadoop.yarn.server.nodemanager.Context;
+import org.apache.hadoop.yarn.server.nodemanager.containermanager.ContainerManager;
import org.apache.hadoop.yarn.server.nodemanager.containermanager.container.Container;
import org.apache.hadoop.yarn.server.nodemanager.containermanager.linux.privileged.PrivilegedOperation;
import org.apache.hadoop.yarn.server.nodemanager.containermanager.linux.privileged.PrivilegedOperationException;
@@ -49,6 +51,8 @@
import org.apache.hadoop.yarn.server.nodemanager.containermanager.resourceplugin.DockerCommandPlugin;
import org.apache.hadoop.yarn.server.nodemanager.containermanager.resourceplugin.ResourcePlugin;
import org.apache.hadoop.yarn.server.nodemanager.containermanager.resourceplugin.ResourcePluginManager;
+import org.apache.hadoop.yarn.server.nodemanager.containermanager.localizer.LocalizedResource;
+import org.apache.hadoop.yarn.server.nodemanager.containermanager.localizer.ResourceLocalizationService;
import org.apache.hadoop.yarn.server.nodemanager.containermanager.runtime.ContainerExecutionException;
import org.apache.hadoop.yarn.server.nodemanager.containermanager.runtime.ContainerRuntimeConstants;
import org.apache.hadoop.yarn.server.nodemanager.containermanager.runtime.ContainerRuntimeContext;
@@ -136,6 +140,7 @@
private Container container;
private ContainerId cId;
private ApplicationAttemptId appAttemptId;
+ private ApplicationId mockApplicationId;
private ContainerLaunchContext context;
private Context nmContext;
private HashMap env;
@@ -165,7 +170,6 @@
private final String whitelistedUser = "yoda";
private String[] testCapabilities;
private final String signalPid = "1234";
- private String runtimeTypeUpper = "DOCKER";
private static final String RUNTIME_TYPE = "DOCKER";
private final static String ENV_OCI_CONTAINER_PID_NAMESPACE =
@@ -201,6 +205,7 @@ public void setup() {
container = mock(Container.class);
cId = mock(ContainerId.class);
appAttemptId = mock(ApplicationAttemptId.class);
+ mockApplicationId = mock(ApplicationId.class);
context = mock(ContainerLaunchContext.class);
env = new HashMap();
env.put("FROM_CLIENT", "1");
@@ -210,6 +215,8 @@ public void setup() {
env.put(DockerLinuxContainerRuntime.ENV_DOCKER_CONTAINER_IMAGE, image);
when(container.getContainerId()).thenReturn(cId);
when(cId.toString()).thenReturn(containerId);
+ when(mockApplicationId.toString()).thenReturn("applicationId");
+ when(appAttemptId.getApplicationId()).thenReturn(mockApplicationId);
when(cId.getApplicationAttemptId()).thenReturn(appAttemptId);
when(container.getLaunchContext()).thenReturn(context);
when(context.getEnvironment()).thenReturn(env);
@@ -280,6 +287,9 @@ public void setup() {
localizedResources.put(new Path("/test_local_dir/test_resource_file"),
Collections.singletonList("test_dir/test_resource_file"));
+ File tmpDir = new File(tmpPath);
+ tmpDir.mkdirs();
+
testCapabilities = new String[] {"NET_BIND_SERVICE", "SYS_CHROOT"};
conf.setStrings(YarnConfiguration.NM_DOCKER_CONTAINER_CAPABILITIES,
testCapabilities);
@@ -328,6 +338,19 @@ public Context createMockNMContext() {
when(mockNMContext.getContainers()).thenReturn(containerMap);
when(containerMap.get(any())).thenReturn(container);
+ ContainerManager mockContainerManager = mock(ContainerManager.class);
+ ResourceLocalizationService mockLocalzationService =
+ mock(ResourceLocalizationService.class);
+
+ LocalizedResource mockLocalizedResource = mock(LocalizedResource.class);
+
+ when(mockLocalizedResource.getLocalPath()).thenReturn(new Path("/local/layer1"));
+ when(mockLocalzationService.getLocalizedResource(any(), anyString(), any()))
+ .thenReturn(mockLocalizedResource);
+ when(mockContainerManager.getResourceLocalizationService())
+ .thenReturn(mockLocalzationService);
+ when(mockNMContext.getContainerManager()).thenReturn(mockContainerManager);
+
try {
when(localDirsHandler.getLocalPathForWrite(anyString()))
.thenReturn(new Path(tmpPath));
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/TestHdfsManifestToResourcesPlugin.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/TestHdfsManifestToResourcesPlugin.java
new file mode 100644
index 00000000000..451d67005c3
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/TestHdfsManifestToResourcesPlugin.java
@@ -0,0 +1,168 @@
+/*
+ * *
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ * /
+ */
+
+package org.apache.hadoop.yarn.server.nodemanager.containermanager.linux.runtime;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FileStatus;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.yarn.api.records.LocalResource;
+import org.apache.hadoop.yarn.api.records.LocalResourceType;
+import org.apache.hadoop.yarn.api.records.LocalResourceVisibility;
+import org.apache.hadoop.yarn.api.records.URL;
+import org.junit.Assert;
+import org.junit.Before;
+import org.junit.Test;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.File;
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.List;
+
+import static org.apache.hadoop.yarn.conf.YarnConfiguration.NM_RUNC_IMAGE_TOPLEVEL_DIR;
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.when;
+
+public class TestHdfsManifestToResourcesPlugin {
+ private static final Logger LOG =
+ LoggerFactory.getLogger(TestHdfsManifestToResourcesPlugin.class);
+ private Configuration conf;
+ private String tmpPath = new StringBuffer(
+ System.getProperty("test.build.data"))
+ .append('/').append("hadoop.tmp.dir").toString();
+ private static final String LAYER_MEDIA_TYPE =
+ "application/vnd.docker.image.rootfs.diff.tar.gzip";
+ private static final String CONFIG_MEDIA_TYPE =
+ "application/vnd.docker.container.image.v1+json";
+
+ @Before
+ public void setup() {
+ conf = new Configuration();
+ File tmpDir = new File(tmpPath);
+ tmpDir.mkdirs();
+ }
+
+ @Test
+ public void testGetLayerResources() throws IOException {
+ ImageManifest mockManifest = mock(ImageManifest.class);
+ ImageManifest.Blob mockLayer1 = mock(ImageManifest.Blob.class);
+ ImageManifest.Blob mockLayer2 = mock(ImageManifest.Blob.class);
+ String digest1Hash =
+ "e060f9dd9e8cd9ec0e2814b661a96d78f7298120d7654ba9f83ebfb11ff1fb1e";
+ String digest2Hash =
+ "5af5ff88469c8473487bfbc2fe81b4e7d84644bd91f1ab9305de47ef5673637e";
+ String digest1 =
+ "sha256:" + digest1Hash;
+ String digest2 =
+ "sha256:" + digest2Hash;
+ long size1 = 1234;
+ long size2 = 5678;
+
+ when(mockLayer1.getMediaType()).thenReturn(LAYER_MEDIA_TYPE);
+ when(mockLayer1.getDigest()).thenReturn(digest1);
+ when(mockLayer1.getSize()).thenReturn(size1);
+
+ when(mockLayer2.getMediaType()).thenReturn(LAYER_MEDIA_TYPE);
+ when(mockLayer2.getDigest()).thenReturn(digest2);
+ when(mockLayer2.getSize()).thenReturn(size2);
+
+ ArrayList mockLayers = new ArrayList<>();
+ mockLayers.add(mockLayer1);
+ mockLayers.add(mockLayer2);
+
+ when(mockManifest.getLayers()).thenReturn(mockLayers);
+
+ conf.set(NM_RUNC_IMAGE_TOPLEVEL_DIR, tmpPath);
+ long modTime = 123456789;
+
+ HdfsManifestToResourcesPlugin hdfsManifestToResourcesPlugin =
+ new HdfsManifestToResourcesPlugin() {
+ @Override
+ protected FileStatus statBlob(Path path) throws IOException {
+ FileStatus mockFileStatus = mock(FileStatus.class);
+ when(mockFileStatus.getModificationTime()).thenReturn(modTime);
+ return mockFileStatus;
+ }
+ };
+ hdfsManifestToResourcesPlugin.init(conf);
+
+ List returnedLayers =
+ hdfsManifestToResourcesPlugin.getLayerResources(mockManifest);
+
+ URL url1 = URL.fromPath(new Path(tmpPath + "/layers",
+ digest1Hash + ".sqsh"));
+ URL url2 = URL.fromPath(new Path(tmpPath + "/layers",
+ digest2Hash + ".sqsh"));
+
+ LocalResource rsrc1 = LocalResource.newInstance(url1,
+ LocalResourceType.FILE, LocalResourceVisibility.PUBLIC,
+ size1, modTime);
+ LocalResource rsrc2 = LocalResource.newInstance(url2,
+ LocalResourceType.FILE, LocalResourceVisibility.PUBLIC,
+ size2, modTime);
+
+ Assert.assertEquals(rsrc1, returnedLayers.get(0));
+ Assert.assertEquals(rsrc2, returnedLayers.get(1));
+
+ }
+
+ @Test
+ public void testGetConfigResources() throws IOException {
+ ImageManifest mockManifest = mock(ImageManifest.class);
+ ImageManifest.Blob mockConfig = mock(ImageManifest.Blob.class);
+ String digestHash =
+ "e23cac476d0238f0f859c1e07e5faad85262bca490ef5c3a9da32a5b39c6b204";
+ String digest =
+ "sha256:" + digestHash;
+ long size = 1234;
+
+ when(mockConfig.getMediaType()).thenReturn(CONFIG_MEDIA_TYPE);
+ when(mockConfig.getDigest()).thenReturn(digest);
+ when(mockConfig.getSize()).thenReturn(size);
+ when(mockManifest.getConfig()).thenReturn(mockConfig);
+
+ conf.set(NM_RUNC_IMAGE_TOPLEVEL_DIR, tmpPath);
+ long modTime = 123456789;
+
+ HdfsManifestToResourcesPlugin hdfsManifestToResourcesPlugin =
+ new HdfsManifestToResourcesPlugin() {
+ @Override
+ protected FileStatus statBlob(Path path) throws IOException {
+ FileStatus mockFileStatus = mock(FileStatus.class);
+ when(mockFileStatus.getModificationTime()).thenReturn(modTime);
+ return mockFileStatus;
+ }
+ };
+ hdfsManifestToResourcesPlugin.init(conf);
+
+ LocalResource returnedLayer =
+ hdfsManifestToResourcesPlugin.getConfigResource(mockManifest);
+
+ URL url1 = URL.fromPath(new Path(tmpPath + "/config", digestHash));
+
+ LocalResource rsrc = LocalResource.newInstance(url1,
+ LocalResourceType.FILE, LocalResourceVisibility.PUBLIC,
+ size, modTime);
+
+ Assert.assertEquals(rsrc, returnedLayer);
+ }
+}
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/TestImageTagToManifestPlugin.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/TestImageTagToManifestPlugin.java
new file mode 100644
index 00000000000..83b22a76ffe
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/TestImageTagToManifestPlugin.java
@@ -0,0 +1,229 @@
+/*
+ * *
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ * /
+ */
+
+package org.apache.hadoop.yarn.server.nodemanager.containermanager.linux.runtime;
+
+import org.apache.hadoop.conf.Configuration;
+import org.codehaus.jackson.map.ObjectMapper;
+import org.junit.Assert;
+import org.junit.Before;
+import org.junit.Test;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.BufferedReader;
+import java.io.File;
+import java.io.IOException;
+import java.io.PrintWriter;
+
+import static org.apache.hadoop.yarn.conf.YarnConfiguration.NM_RUNC_IMAGE_TOPLEVEL_DIR;
+import static org.apache.hadoop.yarn.server.nodemanager.containermanager.linux.runtime.ImageTagToManifestPlugin.NM_HDFS_RUNC_IMAGE_TAG_TO_HASH_FILE;
+import static org.apache.hadoop.yarn.server.nodemanager.containermanager.linux.runtime.ImageTagToManifestPlugin.NM_LOCAL_RUNC_IMAGE_TAG_TO_HASH_FILE;
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.when;
+
+public class TestImageTagToManifestPlugin {
+ private static final Logger LOG =
+ LoggerFactory.getLogger(TestImageTagToManifestPlugin.class);
+ private MockImageTagToManifestPlugin mockImageTagToManifestPlugin;
+ private Configuration conf;
+ private String tmpPath =
+ new StringBuffer(System.getProperty("test.build.data"))
+ .append('/').append("hadoop.tmp.dir").toString();
+ private ObjectMapper mapper;
+
+ private String manifestJson =
+ "{\n" +
+ " \"schemaVersion\": 2,\n" +
+ " \"mediaType\": \"application/vnd.docker.distribution.manifest.v2+json\",\n" +
+ " \"config\": {\n" +
+ " \"mediaType\": \"application/vnd.docker.container.image.v1+json\",\n" +
+ " \"size\": 2857,\n" +
+ " \"digest\": \"sha256:e23cac476d0238f0f859c1e07e5faad85262bca490ef5c3a9da32a5b39c6b204\"\n" +
+ " },\n" +
+ " \"layers\": [\n" +
+ " {\n" +
+ " \"mediaType\": \"application/vnd.docker.image.rootfs.diff.tar.gzip\",\n" +
+ " \"size\": 185784329,\n" +
+ " \"digest\": \"sha256:e060f9dd9e8cd9ec0e2814b661a96d78f7298120d7654ba9f83ebfb11ff1fb1e\"\n" +
+ " },\n" +
+ " {\n" +
+ " \"mediaType\": \"application/vnd.docker.image.rootfs.diff.tar.gzip\",\n" +
+ " \"size\": 10852,\n" +
+ " \"digest\": \"sha256:5af5ff88469c8473487bfbc2fe81b4e7d84644bd91f1ab9305de47ef5673637e\"\n" +
+ " }\n" +
+ " ]\n" +
+ "}";
+
+ @Before
+ public void setup() {
+ conf = new Configuration();
+ mapper = new ObjectMapper();
+ File tmpDir = new File(tmpPath);
+ tmpDir.mkdirs();
+ }
+
+ public class MockImageTagToManifestPlugin extends ImageTagToManifestPlugin {
+ private BufferedReader mockLocalBufferedReader;
+ private BufferedReader mockHdfsBufferedReader;
+
+ MockImageTagToManifestPlugin(BufferedReader mockLocalBufferedReader,
+ BufferedReader mockHdfsBufferedReader) {
+ super();
+ this.mockLocalBufferedReader = mockLocalBufferedReader;
+ this.mockHdfsBufferedReader = mockHdfsBufferedReader;
+ }
+
+ @Override
+ protected BufferedReader getLocalImageToHashReader() throws IOException {
+ return mockLocalBufferedReader;
+ }
+
+ @Override
+ protected BufferedReader getHdfsImageToHashReader() throws IOException {
+ return mockHdfsBufferedReader;
+ }
+ }
+
+
+ @Test
+ public void testLocalGetHashFromImageTag() throws IOException {
+ BufferedReader mockLocalBufferedReader = mock(BufferedReader.class);
+ BufferedReader mockHdfsBufferedReader = mock(BufferedReader.class);
+
+ String commentImage = "commentimage:latest";
+ String commentImageHash =
+ "142fff813433c1faa8796388db3a1fa1e899ba08c9e42ad2e33c42696d0f15d2";
+
+ String fakeImageLatest = "fakeimage:latest";
+ String fakeImageCurrent= "fakeimage:current";
+ String fakeImageHash =
+ "f75903872eb2963e158502ef07f2e56d3a2e90a012b4afe3440461b54142a567";
+
+ String busyboxImage = "repo/busybox:123";
+ String busyboxHash =
+ "c6912b9911deceec6c43ebb4c31c96374a8ebb3de4cd75f377dba6c07707de6e";
+
+ String commentLine = "#" + commentImage + commentImageHash + "#2nd comment";
+ String busyboxLine = busyboxImage + ":" + busyboxHash + "#comment";
+ String fakeImageLine = fakeImageLatest + "," + fakeImageCurrent + ":"
+ + fakeImageHash + "#fakeimage comment";
+
+ when(mockLocalBufferedReader.readLine()).thenReturn(commentLine,
+ fakeImageLine, busyboxLine, null);
+
+ mockImageTagToManifestPlugin = new MockImageTagToManifestPlugin(
+ mockLocalBufferedReader, mockHdfsBufferedReader);
+ mockImageTagToManifestPlugin.loadImageToHashFiles();
+
+ String returnedFakeImageHash = mockImageTagToManifestPlugin
+ .getHashFromImageTag(fakeImageLatest);
+ String returnedBusyboxHash = mockImageTagToManifestPlugin
+ .getHashFromImageTag(busyboxImage);
+ String returnedCommentHash = mockImageTagToManifestPlugin
+ .getHashFromImageTag(commentImage);
+
+ Assert.assertEquals(fakeImageHash, returnedFakeImageHash);
+ Assert.assertEquals(busyboxHash, returnedBusyboxHash);
+
+ //Image hash should not be found, so returned hash should be the tag
+ Assert.assertEquals(commentImage, returnedCommentHash);
+ }
+
+ @Test
+ public void testHdfsGetHashFromImageTag() throws IOException {
+ BufferedReader mockLocalBufferedReader = mock(BufferedReader.class);
+ BufferedReader mockHdfsBufferedReader = mock(BufferedReader.class);
+
+ String commentImage = "commentimage:latest";
+ String commentImageHash =
+ "142fff813433c1faa8796388db3a1fa1e899ba08c9e42ad2e33c42696d0f15d2";
+
+ String fakeImageLatest = "fakeimage:latest";
+ String fakeImageCurrent= "fakeimage:current";
+ String fakeImageHash =
+ "f75903872eb2963e158502ef07f2e56d3a2e90a012b4afe3440461b54142a567";
+
+ String busyboxImage = "repo/busybox:123";
+ String busyboxHash =
+ "c6912b9911deceec6c43ebb4c31c96374a8ebb3de4cd75f377dba6c07707de6e";
+
+ String commentLine = "#" + commentImage + commentImageHash + "#2nd comment";
+ String busyboxLine = busyboxImage + ":" + busyboxHash + "#comment";
+ String fakeImageLine = fakeImageLatest + "," + fakeImageCurrent + ":"
+ + fakeImageHash + "#fakeimage comment";
+
+ when(mockHdfsBufferedReader.readLine()).thenReturn(commentLine,
+ fakeImageLine, busyboxLine, null);
+
+ mockImageTagToManifestPlugin = new MockImageTagToManifestPlugin(
+ mockLocalBufferedReader, mockHdfsBufferedReader);
+ mockImageTagToManifestPlugin.loadImageToHashFiles();
+
+ String returnedFakeImageHash = mockImageTagToManifestPlugin
+ .getHashFromImageTag(fakeImageLatest);
+ String returnedBusyboxHash = mockImageTagToManifestPlugin
+ .getHashFromImageTag(busyboxImage);
+ String returnedCommentHash = mockImageTagToManifestPlugin
+ .getHashFromImageTag(commentImage);
+
+ Assert.assertEquals(fakeImageHash, returnedFakeImageHash);
+ Assert.assertEquals(busyboxHash, returnedBusyboxHash);
+
+ //Image hash should not be found, so returned hash should be the tag
+ Assert.assertEquals(commentImage, returnedCommentHash);
+ }
+
+ @Test
+ public void testGetManifestFromImageTag() throws IOException {
+ String manifestPath = tmpPath + "/manifests";
+ File manifestDir = new File(manifestPath);
+ manifestDir.mkdirs();
+
+ conf.set(NM_LOCAL_RUNC_IMAGE_TAG_TO_HASH_FILE, "local-image-tag-to-hash");
+ conf.set(NM_HDFS_RUNC_IMAGE_TAG_TO_HASH_FILE, "hdfs-image-tag-to-hash");
+ conf.set(NM_RUNC_IMAGE_TOPLEVEL_DIR, tmpPath);
+ String manifestHash =
+ "d0e8c542d28e8e868848aeb58beecb31079eb7ada1293c4bc2eded08daed605a";
+
+ PrintWriter printWriter = new PrintWriter(
+ manifestPath + "/" + manifestHash);
+ printWriter.println(manifestJson);
+ printWriter.close();
+
+ BufferedReader mockLocalBufferedReader = mock(BufferedReader.class);
+ BufferedReader mockHdfsBufferedReader = mock(BufferedReader.class);
+
+ mockImageTagToManifestPlugin = new MockImageTagToManifestPlugin(
+ mockLocalBufferedReader, mockHdfsBufferedReader) {
+ @Override
+ public String getHashFromImageTag(String imageTag) {
+ return manifestHash;
+ }
+ };
+ mockImageTagToManifestPlugin.init(conf);
+
+ ImageManifest manifest = mockImageTagToManifestPlugin
+ .getManifestFromImageTag("image");
+ ImageManifest expectedManifest =
+ mapper.readValue(manifestJson, ImageManifest.class);
+ Assert.assertEquals(expectedManifest.toString(), manifest.toString());
+ }
+}
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/TestRuncContainerRuntime.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/TestRuncContainerRuntime.java
new file mode 100644
index 00000000000..e0df6bb8132
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/TestRuncContainerRuntime.java
@@ -0,0 +1,915 @@
+/*
+ * *
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ * /
+ */
+
+package org.apache.hadoop.yarn.server.nodemanager.containermanager.linux.runtime;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.yarn.api.records.ApplicationAttemptId;
+import org.apache.hadoop.yarn.api.records.ApplicationId;
+import org.apache.hadoop.yarn.api.records.ContainerId;
+import org.apache.hadoop.yarn.api.records.ContainerLaunchContext;
+import org.apache.hadoop.yarn.api.records.LocalResource;
+import org.apache.hadoop.yarn.api.records.LocalResourceType;
+import org.apache.hadoop.yarn.api.records.LocalResourceVisibility;
+import org.apache.hadoop.yarn.api.records.Resource;
+import org.apache.hadoop.yarn.api.records.URL;
+import org.apache.hadoop.yarn.conf.YarnConfiguration;
+import org.apache.hadoop.yarn.server.nodemanager.Context;
+import org.apache.hadoop.yarn.server.nodemanager.containermanager.ContainerManager;
+import org.apache.hadoop.yarn.server.nodemanager.containermanager.container.Container;
+import org.apache.hadoop.yarn.server.nodemanager.containermanager.linux.privileged.PrivilegedOperation;
+import org.apache.hadoop.yarn.server.nodemanager.containermanager.linux.privileged.PrivilegedOperationException;
+import org.apache.hadoop.yarn.server.nodemanager.containermanager.linux.privileged.PrivilegedOperationExecutor;
+import org.apache.hadoop.yarn.server.nodemanager.containermanager.linux.resources.CGroupsHandler;
+import org.apache.hadoop.yarn.server.nodemanager.containermanager.linux.runtime.OCIContainerExecutorConfig.OCIRuntimeConfig;
+import org.apache.hadoop.yarn.server.nodemanager.containermanager.linux.runtime.OCIContainerExecutorConfig.OCILayer;
+import org.apache.hadoop.yarn.server.nodemanager.containermanager.linux.runtime.OCIContainerExecutorConfig.OCIRuntimeConfig.OCIMount;
+import org.apache.hadoop.yarn.server.nodemanager.containermanager.linux.runtime.OCIContainerExecutorConfig.OCIRuntimeConfig.OCIProcessConfig;
+import org.apache.hadoop.yarn.server.nodemanager.containermanager.linux.runtime.RuncContainerRuntime.RuncRuntimeObject;
+import org.apache.hadoop.yarn.server.nodemanager.containermanager.resourceplugin.ResourcePluginManager;
+import org.apache.hadoop.yarn.server.nodemanager.containermanager.localizer.LocalizedResource;
+import org.apache.hadoop.yarn.server.nodemanager.containermanager.localizer.ResourceLocalizationService;
+import org.apache.hadoop.yarn.server.nodemanager.containermanager.runtime.ContainerExecutionException;
+import org.apache.hadoop.yarn.server.nodemanager.containermanager.runtime.ContainerRuntimeConstants;
+import org.apache.hadoop.yarn.server.nodemanager.containermanager.runtime.ContainerRuntimeContext;
+import org.apache.hadoop.yarn.server.nodemanager.LocalDirsHandlerService;
+import org.codehaus.jackson.JsonNode;
+import org.codehaus.jackson.map.ObjectMapper;
+import org.junit.Assert;
+import org.junit.Before;
+import org.junit.Rule;
+import org.junit.Test;
+import org.junit.rules.TemporaryFolder;
+import org.junit.runner.RunWith;
+import org.junit.runners.Parameterized;
+import org.mockito.ArgumentCaptor;
+import org.mockito.Mockito;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.File;
+import java.io.IOException;
+import java.net.URISyntaxException;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Collection;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.concurrent.ConcurrentMap;
+
+import static org.apache.hadoop.yarn.conf.YarnConfiguration.NM_RUNC_DEFAULT_RO_MOUNTS;
+import static org.apache.hadoop.yarn.conf.YarnConfiguration.NM_RUNC_DEFAULT_RW_MOUNTS;
+import static org.apache.hadoop.yarn.conf.YarnConfiguration.NM_RUNC_LAYER_MOUNTS_TO_KEEP;
+import static org.apache.hadoop.yarn.server.nodemanager.containermanager.linux.runtime.LinuxContainerRuntimeConstants.*;
+import static org.mockito.ArgumentMatchers.anyString;
+import static org.mockito.Mockito.any;
+import static org.mockito.Mockito.anyBoolean;
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.times;
+import static org.mockito.Mockito.verify;
+import static org.mockito.Mockito.when;
+
+/**
+ * This class tests the {@link RuncContainerRuntime}.
+ */
+@RunWith(Parameterized.class)
+public class TestRuncContainerRuntime {
+ private static final Logger LOG =
+ LoggerFactory.getLogger(TestRuncContainerRuntime.class);
+ private Configuration conf;
+ private PrivilegedOperationExecutor mockExecutor;
+ private CGroupsHandler mockCGroupsHandler;
+ private String containerId;
+ private Container container;
+ private ContainerId cId;
+ private ApplicationAttemptId appAttemptId;
+ private ApplicationId mockApplicationId;
+ private ContainerLaunchContext context;
+ private Context nmContext;
+ private HashMap env;
+ private String image;
+ private String runAsUser = System.getProperty("user.name");
+ private String user;
+ private String appId;
+ private String containerIdStr;
+ private Path containerWorkDir;
+ private Path nmPrivateContainerScriptPath;
+ private Path nmPrivateTokensPath;
+ private Path nmPrivateKeystorePath;
+ private Path nmPrivateTruststorePath;
+ private Path pidFilePath;
+ private List localDirs;
+ private List logDirs;
+ private List filecacheDirs;
+ private List userFilecacheDirs;
+ private List applicationLocalDirs;
+ private List containerLogDirs;
+ private Map> localizedResources;
+ private String resourcesOptions;
+ private ContainerRuntimeContext.Builder builder;
+ private final String submittingUser = "anakin";
+ private ObjectMapper mapper;
+ private RuncContainerRuntime.RuncRuntimeObject runcRuntimeObject;
+ private LocalResource localResource;
+ private URL mockUrl;
+ private Resource resource;
+ private int layersToKeep;
+ private int cpuShares;
+ private List expectedMounts;
+ private String tmpPath;
+ private LocalResource config;
+ private List layers;
+
+ private RuncImageTagToManifestPlugin mockRuncImageTagToManifestPlugin =
+ mock(ImageTagToManifestPlugin.class);
+ private RuncManifestToResourcesPlugin mockRuncManifestToResourcesPlugin =
+ mock(HdfsManifestToResourcesPlugin.class);
+
+ @Rule
+ public TemporaryFolder tempDir = new TemporaryFolder();
+
+ @Parameterized.Parameters(name = "https={0}")
+ public static Collection